From fdd7e37dfd3046c6ac0cfa9f0b47eb13d5013ebf Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 14:59:15 +0100 Subject: [PATCH 001/144] Add span/expected API, error policy, and UB-free byteswap Co-Authored-By: Claude Opus 4.8 --- docs/coverage.html | 793 ---------------------------------------- include/itch/parser.hpp | 199 +++++++--- 2 files changed, 152 insertions(+), 840 deletions(-) delete mode 100644 docs/coverage.html diff --git a/docs/coverage.html b/docs/coverage.html deleted file mode 100644 index 8d74323..0000000 --- a/docs/coverage.html +++ /dev/null @@ -1,793 +0,0 @@ - - - - - - GCC Code Coverage Report - - - - - - -
-

GCC Code Coverage Report

-
-
-
- - - - - - - - - - - - - -
Directory:src/
Date:2026-01-17 09:41:03
Coverage: - low: ≥ 0% - medium: ≥ 75.0% - high: ≥ 90.0% -
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CoverageExecExclTotal
Lines:91.2%6920759
Functions:88.3%1730196
Branches:53.8%3240602
-
-
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FileLinesFunctionsBranches
- messages.cpp - - 96.7 - 96.7%146 / 0 / 15197.9%47 / 0 / 4850.0%103 / 0 / 206
- order_book.cpp - - 88.5 - 88.5%184 / 0 / 208100.0%36 / 0 / 3659.7%123 / 0 / 206
- parser.cpp - - 90.5 - 90.5%362 / 0 / 40080.4%90 / 0 / 11251.6%98 / 0 / 190
-
-
- - - diff --git a/include/itch/parser.hpp b/include/itch/parser.hpp index 611988f..751da38 100644 --- a/include/itch/parser.hpp +++ b/include/itch/parser.hpp @@ -1,10 +1,21 @@ #pragma once +#include +#include +#include +#include +#include +#include #include #include -#include +#include +#include #include +#ifdef __cpp_lib_expected +#include +#endif + #include "itch/messages.hpp" namespace itch { @@ -15,6 +26,24 @@ namespace itch { /// @param const Message& A const reference to the fully parsed message object. using MessageCallback = std::function; +/// @brief Categories of recoverable problems encountered while framing a feed. +/// +/// These are surfaced through the non-throwing `try_parse` entry points and the +/// optional error callback, so callers can react without relying on exceptions +/// or on the library writing to a global stream. +enum class ParseError { + truncated, ///< The buffer ended in the middle of a frame. + unknown_type, ///< The message type byte does not correspond to a known message. + size_mismatch, ///< The declared length is shorter than the message type requires. +}; + +/// @brief The signature for the optional diagnostics callback. +/// +/// @param ParseError The category of problem that occurred. +/// @param char The offending message type byte (`'\0'` when not applicable, +/// e.g. for a truncated header). +using ErrorCallback = std::function; + /// @brief A high-performance parser for the NASDAQ TotalView-ITCH 5.0 protocol. /// /// This class is designed to parse a raw binary feed of ITCH 5.0 messages, @@ -37,11 +66,9 @@ class Parser { public: /// @brief Constructs a Parser instance. /// - /// The constructor pre-populates an internal dispatch table (a map of - /// handlers) by registering a specific parsing function for each known ITCH - /// message type. This setup makes the parsing process a fast lookup - /// operation at runtime. - Parser(); + /// Dispatch is driven by a compile-time table keyed on the message type + /// byte, so no per-instance setup is required. + Parser() = default; /// @brief Parses messages from a memory buffer and invokes a callback for /// each. @@ -129,58 +156,136 @@ class Parser { /// @throw std::runtime_error on stream reading errors. auto parse(std::istream& data, const std::vector& messages) -> std::vector; + /// @brief Parses messages from a byte span and invokes a callback for each. + /// + /// The `std::span` overloads are the preferred modern interface: the span + /// carries its own size, which prevents the pointer/length desynchronization + /// that the raw `(const char*, size_t)` overloads are prone to. Those raw + /// overloads are retained as thin shims for C interop. + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @param callback A function to be called for each successfully parsed + /// message. + /// @throw std::runtime_error if the buffer ends in the middle of a message. + auto parse(std::span data, const MessageCallback& callback) -> void; + + /// @brief Parses all messages from a byte span and returns them in a vector. + auto parse(std::span data) -> std::vector; + + /// @brief Parses messages from a byte span, keeping only the requested types. + auto parse(std::span data, const std::vector& messages) + -> std::vector; + +#ifdef __cpp_lib_expected + /// @brief Non-throwing parse: invokes a callback per message, reporting + /// truncation through the return value instead of an exception. + /// + /// This is the latency-friendly entry point for callers that prefer not to + /// pay for exceptions on the hot path. Unknown message types and oversized + /// or undersized frames are routed to the diagnostics policy (counters plus + /// the optional error callback) and skipped; only an unrecoverable + /// truncation yields an `unexpected` result. + /// + /// Available when the standard library provides `std::expected` (C++23). + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @param callback A function to be called for each successfully parsed + /// message. + /// @return Nothing on success, or a `ParseError` describing the failure. + [[nodiscard]] auto try_parse(std::span data, const MessageCallback& callback) + -> std::expected; + + /// @brief Non-throwing parse that collects all messages into a vector. + [[nodiscard]] auto try_parse(std::span data) + -> std::expected, ParseError>; +#endif + + /// @brief Registers a callback invoked for each recoverable framing problem. + /// + /// Passing an empty function clears any previously installed callback. The + /// default behavior, with no callback installed, is to silently skip and + /// count the offending frame. + auto set_error_callback(ErrorCallback callback) -> void; + + /// @brief The number of frames skipped because their type byte was unknown. + [[nodiscard]] auto unknown_message_count() const noexcept -> std::uint64_t { + return m_unknown_message_count; + } + + /// @brief The number of frames skipped because their declared length was too + /// small for the message type. + [[nodiscard]] auto malformed_message_count() const noexcept -> std::uint64_t { + return m_malformed_message_count; + } + + /// @brief Resets the accumulating diagnostics counters to zero. + auto reset_diagnostics() noexcept -> void { + m_unknown_message_count = 0; + m_malformed_message_count = 0; + } + private: - using Handler = std::function; - std::map m_handlers; - - /// @brief A template function to register a handler for a specific - /// message type. - /// @tparam T The C++ struct type representing the message (e.g., - /// AddOrderMessage). - /// @param type The character identifier for this message type (e.g., - /// 'A'). - template - auto register_handler(char type) -> void; + /// @brief The shared, non-throwing framing loop backing every parse overload. + /// Returns `std::nullopt` on success, or the `ParseError` that aborted + /// the loop (only an unrecoverable truncation aborts it). + auto parse_impl(const char* data, std::size_t size, const MessageCallback& callback) + -> std::optional; + + /// @brief Records a recoverable framing problem and notifies the callback. + auto report_error(ParseError error, char message_type) -> void; + + ErrorCallback m_error_callback {}; + std::uint64_t m_unknown_message_count {0}; + std::uint64_t m_malformed_message_count {0}; }; namespace utils { -// Generic byte-swapping function for any integral type. -template -T swap_bytes(T value) { - static_assert(std::is_integral_v, "swap_bytes can only be used with integral types"); - union { - T val; - uint8_t bytes[sizeof(T)]; - } src, dst; - src.val = value; - for (size_t i = 0; i < sizeof(T); ++i) { - dst.bytes[i] = src.bytes[sizeof(T) - 1 - i]; +/// @brief Reverses the byte order of an integral value. +/// +/// Prefers `std::byteswap` (C++23) and otherwise falls back to a well-defined +/// `std::bit_cast` based reversal. This deliberately avoids reading an inactive +/// union member, which is undefined behavior in C++ and a hazard on the parser +/// hot path. +template +[[nodiscard]] constexpr auto swap_bytes(IntType value) noexcept -> IntType { +#ifdef __cpp_lib_byteswap + return std::byteswap(value); +#else + if constexpr (sizeof(IntType) == 1) { + return value; + } else { + auto bytes = std::bit_cast>(value); + std::ranges::reverse(bytes); + return std::bit_cast(bytes); } - return dst.val; +#endif } -// Check the system's endianness at compile time (or runtime as a fallback). -// This determines if we need to swap bytes at all. -inline bool is_little_endian(); - -// Converts a value from big-endian (network order) to the host's native byte -// order. On little-endian systems (like x86/x64), it swaps the bytes. On -// big-endian systems, it does nothing. -template -T from_big_endian(T value); +/// @brief Converts a big-endian (network order) value to the host byte order. +/// +/// On little-endian hosts this swaps the bytes; on big-endian hosts it is a +/// no-op. The choice is made at compile time via `std::endian`. +template +[[nodiscard]] constexpr auto from_big_endian(IntType value) noexcept -> IntType { + if constexpr (std::endian::native == std::endian::little) { + return swap_bytes(value); + } else { + return value; + } +} -// Unpacks a value of type T from the buffer at the given offset, updating -// the offset accordingly. Handles endianness conversion for integral types. -template -T unpack(const char* buffer, size_t& offset); +/// @brief Unpacks a value of type T from the buffer, advancing the offset. +/// Handles endianness conversion for multi-byte integral types. +template +auto unpack(const char* buffer, std::size_t& offset) -> ValueType; -// Specialization for char arrays (strings) -inline void unpack_string(const char* buffer, size_t& offset, char* dest, size_t size); +/// @brief Copies a fixed-width character field out of the buffer. +inline auto unpack_string(const char* buffer, std::size_t& offset, char* dest, std::size_t size) + -> void; -// ITCH timestamps are 48-bit big-endian integers. -// 2 bytes for high part, 4 bytes for low part. -inline uint64_t unpack_timestamp(const char* buffer, size_t& offset); +/// @brief Unpacks a 48-bit big-endian ITCH timestamp (2-byte high, 4-byte low). +inline auto unpack_timestamp(const char* buffer, std::size_t& offset) -> std::uint64_t; } // namespace utils } // namespace itch From 49aeba67f85b80ac87ebdb9833bd0d83225b859c Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 14:59:15 +0100 Subject: [PATCH 002/144] Use flat dispatch table with length validation and error policy Co-Authored-By: Claude Opus 4.8 --- src/parser.cpp | 286 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 198 insertions(+), 88 deletions(-) diff --git a/src/parser.cpp b/src/parser.cpp index b74e10e..9a90e7b 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1,68 +1,48 @@ #include "itch/parser.hpp" +#include +#include #include #include #include #include +#include #include namespace itch { namespace utils { -inline auto is_little_endian() -> bool { -#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - return true; -#elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return false; -#elif defined(_WIN32) // Windows is always little-endian - return true; -#else - // Runtime check if compile-time check is not available - const union { - uint32_t i; - char c[4]; - } bint = {0x01020304}; - return bint.c[0] == 4; -#endif -} -template -auto from_big_endian(T value) -> T { - if (is_little_endian()) { - return swap_bytes(value); - } - return value; -} +template +auto unpack(const char* buffer, std::size_t& offset) -> ValueType { + ValueType value; + std::memcpy(&value, buffer + offset, sizeof(ValueType)); + offset += sizeof(ValueType); -template -auto unpack(const char* buffer, size_t& offset) -> T { - T value; - std::memcpy(&value, buffer + offset, sizeof(T)); - offset += sizeof(T); - - if constexpr (std::is_integral_v && sizeof(T) > 1) { + if constexpr (std::is_integral_v && sizeof(ValueType) > 1) { return from_big_endian(value); } else { return value; } } -inline auto unpack_string(const char* buffer, size_t& offset, char* dest, size_t size) -> void { +inline auto unpack_string(const char* buffer, std::size_t& offset, char* dest, std::size_t size) + -> void { std::memcpy(dest, buffer + offset, size); offset += size; } -inline auto unpack_timestamp(const char* buffer, size_t& offset) -> uint64_t { - uint16_t high {}; - uint32_t low {}; - constexpr int lower_shift = 32; +inline auto unpack_timestamp(const char* buffer, std::size_t& offset) -> std::uint64_t { + std::uint16_t high {}; + std::uint32_t low {}; + constexpr int LOWER_SHIFT = 32; std::memcpy(&high, buffer + offset, sizeof(high)); offset += sizeof(high); std::memcpy(&low, buffer + offset, sizeof(low)); offset += sizeof(low); high = from_big_endian(high); low = from_big_endian(low); - return (static_cast(high) << lower_shift) | low; + return (static_cast(high) << LOWER_SHIFT) | low; } } // namespace utils @@ -246,83 +226,163 @@ auto unpack_message(DLCRMessage& msg, const char* buffer, size_t& offset) -> voi msg.upper_price_range_collar = utils::unpack(buffer, offset); } -template -auto Parser::register_handler(char type) -> void { - m_handlers[type] = [](const char* buffer) -> Message { - T msg; - size_t offset = 1; // Skip message type +namespace { - msg.stock_locate = utils::unpack(buffer, offset); - msg.tracking_number = utils::unpack(buffer, offset); - msg.timestamp = utils::unpack_timestamp(buffer, offset); +// The on-wire ITCH timestamp is 48 bits (6 bytes), but every message struct +// stores it in a 64-bit field. Each struct therefore occupies exactly two bytes +// more than its on-wire encoding, since the timestamp is the only field whose +// storage width differs from its wire width. +constexpr std::size_t TIMESTAMP_STRUCT_PADDING = sizeof(std::uint64_t) - 6; - unpack_message(msg, buffer, offset); +/// @brief The exact on-wire size, in bytes, of a fully formed message of type T. +template +constexpr std::size_t WIRE_SIZE = sizeof(MsgType) - TIMESTAMP_STRUCT_PADDING; - return msg; - }; +// Lock the padding assumption to the spec lengths so a future struct change +// that breaks the derivation is caught at compile time rather than at runtime. +static_assert(WIRE_SIZE == 12); +static_assert(WIRE_SIZE == 39); +static_assert(WIRE_SIZE == 36); +static_assert(WIRE_SIZE == 50); +static_assert(WIRE_SIZE == 48); + +/// @brief Decodes the common header and type-specific body of a message of +/// type MsgType from a frame, returning it wrapped in the Message variant. +template +auto decode_typed(const char* buffer) -> Message { + MsgType msg; + std::size_t offset = 1; // Skip the message type byte. + + msg.stock_locate = utils::unpack(buffer, offset); + msg.tracking_number = utils::unpack(buffer, offset); + msg.timestamp = utils::unpack_timestamp(buffer, offset); + + unpack_message(msg, buffer, offset); + + return msg; } -Parser::Parser() { - register_handler('S'); - register_handler('R'); - register_handler('H'); - register_handler('Y'); - register_handler('L'); - register_handler('V'); - register_handler('W'); - register_handler('K'); - register_handler('J'); - register_handler('h'); - register_handler('A'); - register_handler('F'); - register_handler('E'); - register_handler('C'); - register_handler('X'); - register_handler('D'); - register_handler('U'); - register_handler('P'); - register_handler('Q'); - register_handler('B'); - register_handler('I'); - register_handler('N'); - register_handler('O'); +/// @brief One slot of the flat, type-byte-indexed dispatch table. +struct DispatchEntry { + std::uint16_t wire_size {0}; ///< Expected on-wire frame size (0 == unknown type). + Message (*decode)(const char*) {nullptr}; ///< Decoder, or null for an unknown type. +}; + +constexpr std::size_t DISPATCH_TABLE_SIZE = 256; + +/// @brief Builds the dispatch table at compile time so message dispatch is a +/// single flat-array lookup plus a direct (inlinable) call, with no map +/// traversal and no type-erased `std::function` indirection. +consteval auto build_dispatch_table() -> std::array { + std::array table {}; + + auto add = [&table](char type) { + table[static_cast(type)] = + DispatchEntry {static_cast(WIRE_SIZE), &decode_typed}; + }; + + add.operator()('S'); + add.operator()('R'); + add.operator()('H'); + add.operator()('Y'); + add.operator()('L'); + add.operator()('V'); + add.operator()('W'); + add.operator()('K'); + add.operator()('J'); + add.operator()('h'); + add.operator()('A'); + add.operator()('F'); + add.operator()('E'); + add.operator()('C'); + add.operator()('X'); + add.operator()('D'); + add.operator()('U'); + add.operator()('P'); + add.operator()('Q'); + add.operator()('B'); + add.operator()('I'); + add.operator()('N'); + add.operator()('O'); + + return table; +} + +constexpr auto DISPATCH_TABLE = build_dispatch_table(); + +} // namespace + +auto Parser::report_error(ParseError error, char message_type) -> void { + switch (error) { + case ParseError::unknown_type: + ++m_unknown_message_count; + break; + case ParseError::size_mismatch: + case ParseError::truncated: + ++m_malformed_message_count; + break; + } + if (m_error_callback) { + m_error_callback(error, message_type); + } } -auto Parser::parse(const char* data, size_t size, const MessageCallback& callback) -> void { - size_t offset = 0; +auto Parser::parse_impl(const char* data, std::size_t size, const MessageCallback& callback) + -> std::optional { + std::size_t offset = 0; while (offset < size) { - // Ensure we can read the length field - if (offset + sizeof(uint16_t) > size) { - throw std::runtime_error("Incomplete message header at end of buffer."); + // Ensure we can read the 2-byte length prefix. + if (offset + sizeof(std::uint16_t) > size) { + report_error(ParseError::truncated, '\0'); + return ParseError::truncated; } - uint16_t length {}; + std::uint16_t length {}; std::memcpy(&length, data + offset, sizeof(length)); length = utils::from_big_endian(length); - offset += sizeof(uint16_t); + offset += sizeof(std::uint16_t); if (length == 0) { - continue; + continue; // Skip zero-length padding frames. } - // Ensure the full message payload is available + // Ensure the full declared payload is present. if (offset + length > size) { - throw std::runtime_error("Incomplete message at end of buffer."); + report_error(ParseError::truncated, '\0'); + return ParseError::truncated; } + const char* message = data + offset; const char message_type = message[0]; + offset += length; - auto handler_it = m_handlers.find(message_type); - if (handler_it != m_handlers.end()) { - callback(handler_it->second(message)); - } else { - std::cerr << "Unknown or unhandled message type: " << message_type << '\n'; + const DispatchEntry& entry = DISPATCH_TABLE[static_cast(message_type)]; + if (entry.decode == nullptr) { + // Unknown type: skip and count rather than writing to a global stream. + report_error(ParseError::unknown_type, message_type); + continue; } - offset += length; + // A frame shorter than the type requires would make the decoder read into + // the next frame; reject it. A longer frame is tolerated for forward + // compatibility (the trailing bytes are simply skipped). + if (length < entry.wire_size) { + report_error(ParseError::size_mismatch, message_type); + continue; + } + + callback(entry.decode(message)); + } + return std::nullopt; +} + +auto Parser::parse(const char* data, size_t size, const MessageCallback& callback) -> void { + if (parse_impl(data, size, callback).has_value()) { + throw std::runtime_error("Incomplete message at end of buffer."); } } constexpr size_t average_message_size = 20; -auto Parser::parse(const char* data, size_t size) -> std::vector { + +auto Parser::parse(const char* data, size_t size) -> std::vector { std::vector messages; messages.reserve(size / average_message_size); parse(data, size, [&](const Message& msg) { messages.push_back(msg); }); @@ -349,6 +409,56 @@ auto Parser::parse(const char* data, size_t size, const std::vector& messa return results; } +namespace { +// std::byte and char may legitimately alias; route through void* to obtain the +// char view the framing core works with, without a forbidden reinterpret_cast. +auto as_char_ptr(std::span data) -> const char* { + // char may alias std::byte; the house style forbids reinterpret_cast, so the + // void* hop is the intended form here. + const void* raw = data.data(); + return static_cast(raw); // NOLINT(bugprone-casting-through-void) +} +} // namespace + +auto Parser::parse(std::span data, const MessageCallback& callback) -> void { + parse(as_char_ptr(data), data.size(), callback); +} + +auto Parser::parse(std::span data) -> std::vector { + return parse(as_char_ptr(data), data.size()); +} + +auto Parser::parse(std::span data, const std::vector& messages) + -> std::vector { + return parse(as_char_ptr(data), data.size(), messages); +} + +#ifdef __cpp_lib_expected +auto Parser::try_parse(std::span data, const MessageCallback& callback) + -> std::expected { + if (auto error = parse_impl(as_char_ptr(data), data.size(), callback)) { + return std::unexpected(*error); + } + return {}; +} + +auto Parser::try_parse(std::span data) + -> std::expected, ParseError> { + std::vector messages; + messages.reserve(data.size() / average_message_size); + if (auto error = parse_impl(as_char_ptr(data), data.size(), [&](const Message& msg) { + messages.push_back(msg); + })) { + return std::unexpected(*error); + } + return messages; +} +#endif + +auto Parser::set_error_callback(ErrorCallback callback) -> void { + m_error_callback = std::move(callback); +} + // Read the whole stream into a buffer static auto read_stream_into_buffer(std::istream& data) -> std::vector { data.seekg(0, std::ios::end); From e57e6fe0cb045363508948c12b680ceef6fd525c Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 14:59:15 +0100 Subject: [PATCH 003/144] Rename to_string parameter for clarity Co-Authored-By: Claude Opus 4.8 --- include/itch/messages.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/itch/messages.hpp b/include/itch/messages.hpp index 826d82b..8050789 100644 --- a/include/itch/messages.hpp +++ b/include/itch/messages.hpp @@ -329,12 +329,12 @@ constexpr double PRICE_DIVISOR = 10000.0; constexpr double MWCB_PRICE_DIVISOR = 1.0E8; // Convert char arrays to strings, trimming trailing spaces. -inline auto to_string(const char* arr, size_t size) -> std::string { +inline auto to_string(const char* source, size_t size) -> std::string { size_t len = size; - while (len > 0 && (arr[len - 1] == ' ' || arr[len - 1] == '\0')) { + while (len > 0 && (source[len - 1] == ' ' || source[len - 1] == '\0')) { len--; } - return std::string {arr, len}; + return std::string {source, len}; } auto print_impl(std::ostream& out, const SystemEventMessage& msg) -> void; From 2a48f11b65b38a88c640f1528cd7a48c73b98faf Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 14:59:16 +0100 Subject: [PATCH 004/144] Add typed fixed-point Price Co-Authored-By: Claude Opus 4.8 --- include/itch/price.hpp | 94 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 include/itch/price.hpp diff --git a/include/itch/price.hpp b/include/itch/price.hpp new file mode 100644 index 0000000..afd33a0 --- /dev/null +++ b/include/itch/price.hpp @@ -0,0 +1,94 @@ +#pragma once + +#include +#include +#include +#include + +namespace itch { + +/// @brief A strongly typed fixed-point price carrying its own decimal scale. +/// +/// ITCH prices are wire integers with an implied number of decimal places. +/// Standard price fields imply 4 decimals (divisor 10,000); MWCB decline-level +/// prices imply 8 decimals (divisor 100,000,000). Exposing both as a bare +/// `uint32_t`/`uint64_t` makes it trivially easy to divide by the wrong scale +/// and silently produce a wrong price. +/// +/// `BasicPrice` encodes the scale in the type. The two scales are therefore +/// distinct types (`StandardPrice` vs `MwcbPrice`) that cannot be mixed, +/// compared, or assigned to one another by accident: the 4-vs-8 decimal mistake +/// becomes a compile error rather than a runtime bug. +/// +/// @tparam RawType The unsigned integer type of the on-wire value. +/// @tparam Decimals The number of implied decimal places. +template +class BasicPrice { + static_assert(std::is_unsigned_v, "Price raw storage must be an unsigned integer"); + + public: + using raw_type = RawType; + + /// @brief The number of implied decimal places for this price scale. + static constexpr unsigned int decimals = Decimals; + + constexpr BasicPrice() noexcept = default; + + /// @brief Wraps a raw on-wire price value at this scale. + explicit constexpr BasicPrice(raw_type raw_value) noexcept : m_raw {raw_value} {} + + /// @brief The underlying raw integer value, exactly as it appears on the wire. + [[nodiscard]] constexpr auto raw() const noexcept -> raw_type { return m_raw; } + + /// @brief The scale's divisor, i.e. 10^Decimals. + [[nodiscard]] static constexpr auto divisor() noexcept -> raw_type { + raw_type result {1}; + for (unsigned int exponent {0}; exponent < Decimals; ++exponent) { + result *= 10; + } + return result; + } + + /// @brief The price as a floating-point value (raw / 10^Decimals). + [[nodiscard]] constexpr auto to_double() const noexcept -> double { + return static_cast(m_raw) / static_cast(divisor()); + } + + /// @brief The price as a fixed-precision decimal string. + [[nodiscard]] auto to_string() const -> std::string { + return std::format("{:.{}f}", to_double(), Decimals); + } + + [[nodiscard]] friend constexpr auto operator==(BasicPrice, BasicPrice) noexcept + -> bool = default; + [[nodiscard]] friend constexpr auto operator<=>(BasicPrice, BasicPrice) noexcept = default; + + private: + raw_type m_raw {0}; +}; + +/// @brief Price scale for every ITCH price field except MWCB decline levels. +using StandardPrice = BasicPrice; + +/// @brief Price scale for MWCB decline-level prices (8 implied decimals). +using MwcbPrice = BasicPrice; + +/// @brief Wraps a raw 4-decimal price value in the typed StandardPrice. +[[nodiscard]] constexpr auto make_price(std::uint32_t raw_value) noexcept -> StandardPrice { + return StandardPrice {raw_value}; +} + +/// @brief Wraps a raw 8-decimal MWCB price value in the typed MwcbPrice. +[[nodiscard]] constexpr auto make_mwcb_price(std::uint64_t raw_value) noexcept -> MwcbPrice { + return MwcbPrice {raw_value}; +} + +} // namespace itch + +/// @brief Formats a price with its scale's number of decimals, e.g. "150.0000". +template +struct std::formatter> : std::formatter { + auto format(itch::BasicPrice price, std::format_context& ctx) const { + return std::formatter::format(price.to_double(), ctx); + } +}; From 38c54e45a48b5acd7480a03ca9ed45fb61232f76 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 14:59:16 +0100 Subject: [PATCH 005/144] Add timestamp-to-wall-clock helpers Co-Authored-By: Claude Opus 4.8 --- include/itch/time.hpp | 50 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 include/itch/time.hpp diff --git a/include/itch/time.hpp b/include/itch/time.hpp new file mode 100644 index 0000000..3976c22 --- /dev/null +++ b/include/itch/time.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include +#include + +namespace itch { + +/// @brief A system-clock time point at nanosecond resolution. +using ItchTimePoint = std::chrono::sys_time; + +/// @brief Combines a session date with an ITCH timestamp into a time point. +/// +/// ITCH timestamps are raw nanoseconds elapsed since midnight, with no date or +/// time-zone attached. This helper anchors that offset to the midnight of a +/// caller-supplied session date. No time-zone conversion is performed: the +/// result is expressed in the same frame as the date you pass in (pass a UTC +/// date for a UTC result, an Eastern-time date for an Eastern result). +/// +/// @param session_date The calendar date whose midnight anchors the timestamp. +/// @param nanos_past_midnight The raw ITCH timestamp (nanoseconds past midnight). +/// @return The absolute time point for the event. +[[nodiscard]] constexpr auto to_time_point( + std::chrono::year_month_day session_date, std::uint64_t nanos_past_midnight +) noexcept -> ItchTimePoint { + return std::chrono::sys_days {session_date} + std::chrono::nanoseconds {nanos_past_midnight}; +} + +/// @brief Formats a raw ITCH timestamp as a time-of-day string. +/// +/// @param nanos_past_midnight The raw ITCH timestamp (nanoseconds past midnight). +/// @return A string of the form "HH:MM:SS.nnnnnnnnn". +[[nodiscard]] inline auto format_timestamp(std::uint64_t nanos_past_midnight) -> std::string { + const std::chrono::hh_mm_ss time_of_day {std::chrono::nanoseconds {nanos_past_midnight}}; + return std::format("{:%T}", time_of_day); +} + +/// @brief Formats a session date plus an ITCH timestamp as a full date-time. +/// +/// @param session_date The calendar date whose midnight anchors the timestamp. +/// @param nanos_past_midnight The raw ITCH timestamp (nanoseconds past midnight). +/// @return A string of the form "YYYY-MM-DD HH:MM:SS.nnnnnnnnn". +[[nodiscard]] inline auto format_time_point( + std::chrono::year_month_day session_date, std::uint64_t nanos_past_midnight +) -> std::string { + return std::format("{:%F %T}", to_time_point(session_date, nanos_past_midnight)); +} + +} // namespace itch From 85fb280ea81ce193ad252c97e8a68e99468879ba Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 14:59:16 +0100 Subject: [PATCH 006/144] Replace map globals with constexpr frozen lookups Co-Authored-By: Claude Opus 4.8 --- include/itch/indicators.hpp | 370 +++++++++++++++++++++--------------- 1 file changed, 212 insertions(+), 158 deletions(-) diff --git a/include/itch/indicators.hpp b/include/itch/indicators.hpp index edb4e5a..c2635c2 100644 --- a/include/itch/indicators.hpp +++ b/include/itch/indicators.hpp @@ -1,23 +1,68 @@ #pragma once -#include -#include +#include +#include +#include #include +#include namespace itch { namespace indicators { -using sv = std::string_view; -const std::map SYSTEM_EVENT_CODES = { +/// @brief An immutable, compile-time lookup table from a key to a description. +/// +/// Replaces the namespace-scope `std::map` globals that this header used to +/// define: those incurred dynamic initialization in every translation unit and +/// risked one-definition-rule trouble. A `FrozenMap` is a literal type held in +/// read-only storage. Its entries are sorted once at compile time, so lookups +/// are a branch-light binary search with no allocation and no runtime setup. +/// +/// @tparam KeyType The key type (`char` for single-letter codes, or +/// `std::string_view` for multi-character codes). +/// @tparam Size The number of entries. +template +class FrozenMap { + public: + using value_type = std::pair; + + constexpr explicit FrozenMap(std::array entries) noexcept + : m_entries {entries} { + std::ranges::sort(m_entries, {}, &value_type::first); + } + + /// @brief Returns the description for a key, or `std::nullopt` if absent. + [[nodiscard]] constexpr auto find(const KeyType& key) const noexcept + -> std::optional { + const auto iter = std::ranges::lower_bound(m_entries, key, {}, &value_type::first); + if (iter != m_entries.end() && iter->first == key) { + return iter->second; + } + return std::nullopt; + } + + /// @brief Returns the description for a key, or a fallback if absent. + [[nodiscard]] constexpr auto at_or(const KeyType& key, std::string_view fallback) const noexcept + -> std::string_view { + return find(key).value_or(fallback); + } + + private: + std::array m_entries; +}; + +template +FrozenMap(std::array, Size>) -> FrozenMap; + +inline constexpr FrozenMap SYSTEM_EVENT_CODES {std::to_array>({ {'O', "Start of Messages"}, {'S', "Start of System hours"}, {'Q', "Start of Market hours"}, {'M', "End of Market hours"}, {'E', "End of System hours"}, {'C', "End of Messages"}, -}; +})}; -const std::map MARKET_CATEGORY = { +inline constexpr FrozenMap MARKET_CATEGORY {std::to_array>({ {'N', "NYSE"}, {'A', "AMEX"}, {'P', "Arca"}, @@ -27,180 +72,189 @@ const std::map MARKET_CATEGORY = { {'Z', "BATS"}, {'V', "Investors Exchange"}, {' ', "Not available"}, -}; +})}; -const std::map FINANCIAL_STATUS_INDICATOR = { - {'D', "Deficient"}, - {'E', "Delinquent"}, - {'Q', "Bankrupt"}, - {'S', "Suspended"}, - {'G', "Deficient and Bankrupt"}, - {'H', "Deficient and Delinquent"}, - {'J', "Delinquent and Bankrupt"}, - {'K', "Deficient, Delinquent and Bankrupt"}, - {'C', "Creations and/or Redemptions Suspended for Exchange Traded Product"}, - {'N', "Normal (Default): Issuer is NOT Deficient, Delinquent, or Bankrupt"}, - {' ', "Not available. Firms should refer to SIAC feeds for code if needed"}, +inline constexpr FrozenMap FINANCIAL_STATUS_INDICATOR { + std::to_array>({ + {'D', "Deficient"}, + {'E', "Delinquent"}, + {'Q', "Bankrupt"}, + {'S', "Suspended"}, + {'G', "Deficient and Bankrupt"}, + {'H', "Deficient and Delinquent"}, + {'J', "Delinquent and Bankrupt"}, + {'K', "Deficient, Delinquent and Bankrupt"}, + {'C', "Creations and/or Redemptions Suspended for Exchange Traded Product"}, + {'N', "Normal (Default): Issuer is NOT Deficient, Delinquent, or Bankrupt"}, + {' ', "Not available. Firms should refer to SIAC feeds for code if needed"}, + }) }; -const std::map ISSUE_CLASSIFICATION_VALUES = { - {'A', "American Depositary Share"}, - {'B', "Bond"}, - {'C', "Common Stock"}, - {'F', "Depository Receipt"}, - {'I', "144A"}, - {'L', "Limited Partnership"}, - {'N', "Notes"}, - {'O', "Ordinary Share"}, - {'P', "Preferred Stock"}, - {'Q', "Other Securities"}, - {'R', "Right"}, - {'S', "Shares of Beneficial Interest"}, - {'T', "Convertible Debenture"}, - {'U', "Unit"}, - {'V', "Units/Beneficial Interest"}, - {'W', "Warrant"}, +inline constexpr FrozenMap ISSUE_CLASSIFICATION_VALUES { + std::to_array>({ + {'A', "American Depositary Share"}, + {'B', "Bond"}, + {'C', "Common Stock"}, + {'F', "Depository Receipt"}, + {'I', "144A"}, + {'L', "Limited Partnership"}, + {'N', "Notes"}, + {'O', "Ordinary Share"}, + {'P', "Preferred Stock"}, + {'Q', "Other Securities"}, + {'R', "Right"}, + {'S', "Shares of Beneficial Interest"}, + {'T', "Convertible Debenture"}, + {'U', "Unit"}, + {'V', "Units/Beneficial Interest"}, + {'W', "Warrant"}, + }) }; -const std::map ISSUE_SUB_TYPE_VALUES = { - {"A", "Preferred Trust Securities"}, - {"AI", "Alpha Index ETNs"}, - {"B", "Index Based Derivative"}, - {"C", "Common Shares"}, - {"CB", "Commodity Based Trust Shares"}, - {"CF", "Commodity Futures Trust Shares"}, - {"CL", "Commodity-Linked Securities"}, - {"CM", "Commodity Index Trust Shares"}, - {"CO", "Collateralized Mortgage Obligation"}, - {"CT", "Currency Trust Shares"}, - {"CU", "Commodity-Currency-Linked Securities"}, - {"CW", "Currency Warrants"}, - {"D", "Global Depositary Shares"}, - {"E", "ETF-Portfolio Depositary Receipt"}, - {"EG", "Equity Gold Shares"}, - {"EI", "ETN-Equity Index-Linked Securities"}, - {"EM", "NextShares Exchange Traded Managed Fund*"}, - {"EN", "Exchange Traded Notes"}, - {"EU", "Equity Units"}, - {"F", "HOLDRS"}, - {"FI", "ETN-Fixed Income-Linked Securities"}, - {"FL", "ETN-Futures-Linked Securities"}, - {"G", "Global Shares"}, - {"I", "ETF-Index Fund Shares"}, - {"IR", "Interest Rate"}, - {"IW", "Index Warrant"}, - {"IX", "Index-Linked Exchangeable Notes"}, - {"J", "Corporate Backed Trust Security"}, - {"L", "Contingent Litigation Right"}, - {"LL", "Limited Liability Company (LLC)"}, - {"M", "Equity-Based Derivative"}, - {"MF", "Managed Fund Shares"}, - {"ML", "ETN-Multi-Factor Index-Linked Securities"}, - {"MT", "Managed Trust Securities"}, - {"N", "NY Registry Shares"}, - {"O", "Open Ended Mutual Fund"}, - {"P", "Privately Held Security"}, - {"PP", "Poison Pill"}, - {"PU", "Partnership Units"}, - {"Q", "Closed-End Funds"}, - {"R", "Reg-S"}, - {"RC", "Commodity-Redeemable Commodity-Linked Securities"}, - {"RF", "ETN-Redeemable Futures-Linked Securities"}, - {"RT", "REIT"}, - {"RU", "Commodity-Redeemable Currency-Linked Securities"}, - {"S", "SEED"}, - {"SC", "Spot Rate Closing"}, - {"SI", "Spot Rate Intraday"}, - {"T", "Tracking Stock"}, - {"TC", "Trust Certificates"}, - {"TU", "Trust Units"}, - {"U", "Portal"}, - {"V", "Contingent Value Right"}, - {"W", "Trust Issued Receipts"}, - {"WC", "World Currency Option"}, - {"X", "Trust"}, - {"Y", "Other"}, - {"Z", "Not Applicable"}, - +inline constexpr FrozenMap ISSUE_SUB_TYPE_VALUES { + std::to_array>({ + {"A", "Preferred Trust Securities"}, + {"AI", "Alpha Index ETNs"}, + {"B", "Index Based Derivative"}, + {"C", "Common Shares"}, + {"CB", "Commodity Based Trust Shares"}, + {"CF", "Commodity Futures Trust Shares"}, + {"CL", "Commodity-Linked Securities"}, + {"CM", "Commodity Index Trust Shares"}, + {"CO", "Collateralized Mortgage Obligation"}, + {"CT", "Currency Trust Shares"}, + {"CU", "Commodity-Currency-Linked Securities"}, + {"CW", "Currency Warrants"}, + {"D", "Global Depositary Shares"}, + {"E", "ETF-Portfolio Depositary Receipt"}, + {"EG", "Equity Gold Shares"}, + {"EI", "ETN-Equity Index-Linked Securities"}, + {"EM", "NextShares Exchange Traded Managed Fund*"}, + {"EN", "Exchange Traded Notes"}, + {"EU", "Equity Units"}, + {"F", "HOLDRS"}, + {"FI", "ETN-Fixed Income-Linked Securities"}, + {"FL", "ETN-Futures-Linked Securities"}, + {"G", "Global Shares"}, + {"I", "ETF-Index Fund Shares"}, + {"IR", "Interest Rate"}, + {"IW", "Index Warrant"}, + {"IX", "Index-Linked Exchangeable Notes"}, + {"J", "Corporate Backed Trust Security"}, + {"L", "Contingent Litigation Right"}, + {"LL", "Limited Liability Company (LLC)"}, + {"M", "Equity-Based Derivative"}, + {"MF", "Managed Fund Shares"}, + {"ML", "ETN-Multi-Factor Index-Linked Securities"}, + {"MT", "Managed Trust Securities"}, + {"N", "NY Registry Shares"}, + {"O", "Open Ended Mutual Fund"}, + {"P", "Privately Held Security"}, + {"PP", "Poison Pill"}, + {"PU", "Partnership Units"}, + {"Q", "Closed-End Funds"}, + {"R", "Reg-S"}, + {"RC", "Commodity-Redeemable Commodity-Linked Securities"}, + {"RF", "ETN-Redeemable Futures-Linked Securities"}, + {"RT", "REIT"}, + {"RU", "Commodity-Redeemable Currency-Linked Securities"}, + {"S", "SEED"}, + {"SC", "Spot Rate Closing"}, + {"SI", "Spot Rate Intraday"}, + {"T", "Tracking Stock"}, + {"TC", "Trust Certificates"}, + {"TU", "Trust Units"}, + {"U", "Portal"}, + {"V", "Contingent Value Right"}, + {"W", "Trust Issued Receipts"}, + {"WC", "World Currency Option"}, + {"X", "Trust"}, + {"Y", "Other"}, + {"Z", "Not Applicable"}, + }) }; -const std::map TRADING_ACTION_REASON_CODES = { - {"T1", "Halt News Pending"}, - {"T2", "Halt News Disseminated"}, - {"T3", "News and Resumption Times"}, - {"T5", "Single Security Trading Pause In Effect"}, - {"T6", "Regulatory Halt - Extraordinary Market Activity"}, - {"T7", "Single Security Trading Pause / Quotation Only Period"}, - {"T8", "Halt ETF"}, - {"T12", "Trading Halted; For Information Requested by Listing Market"}, - {"H4", "Halt Non-Compliance"}, - {"H9", "Halt Filings Not Current"}, - {"H10", "Halt SEC Trading Suspension"}, - {"H11", "Halt Regulatory Concern"}, - {"O1", "Operations Halt; Contact Market Operations"}, - {"LUDP", "Volatility Trading Pause"}, - {"LUDS", "Volatility Trading Pause - Straddle Condition"}, - {"MWC0", "Market Wide Circuit Breaker Halt - Carry over from previous day"}, - {"MWC1", "Market Wide Circuit Breaker Halt - Level 1"}, - {"MWC2", "Market Wide Circuit Breaker Halt - Level 2"}, - {"MWC3", "Market Wide Circuit Breaker Halt - Level 3"}, - {"MWCQ", "Market Wide Circuit Breaker Resumption"}, - {"IPO1", "IPO Issue Not Yet Trading"}, - {"IPOQ", "IPO Security Released for Quotation (Nasdaq Securities Only)"}, - {"IPOE", "IPO Security — Positioning Window Extension (Nasdaq Securities Only)"}, - {"M1", "Corporate Action"}, - {"M2", "Quotation Not Available"}, - {"R1", "New Issue Available"}, - {"R2", "Issue Available"}, - {"R4", "Qualifications Issues Reviewed/Resolved; Quotations/Trading to Resume"}, - {"R9", "Filing Requirements Satisfied/Resolved; Quotations/Trading To Resume"}, - {"C3", "Issuer News Not Forthcoming; Quotations/Trading To Resume"}, - {"C4", "Qualifications Halt Ended; Maintenance Requirements Met; Resume"}, - {"C9", "Qualifications Halt Concluded; Filings Met; Quotes/Trades To Resume"}, - {"C11", - "Trade Halt Concluded By Other Regulatory Authority; Quotes/Trades " - "Resume"}, - {" ", "Reason Not Available"}, +inline constexpr FrozenMap TRADING_ACTION_REASON_CODES { + std::to_array>({ + {"T1", "Halt News Pending"}, + {"T2", "Halt News Disseminated"}, + {"T3", "News and Resumption Times"}, + {"T5", "Single Security Trading Pause In Effect"}, + {"T6", "Regulatory Halt - Extraordinary Market Activity"}, + {"T7", "Single Security Trading Pause / Quotation Only Period"}, + {"T8", "Halt ETF"}, + {"T12", "Trading Halted; For Information Requested by Listing Market"}, + {"H4", "Halt Non-Compliance"}, + {"H9", "Halt Filings Not Current"}, + {"H10", "Halt SEC Trading Suspension"}, + {"H11", "Halt Regulatory Concern"}, + {"O1", "Operations Halt; Contact Market Operations"}, + {"LUDP", "Volatility Trading Pause"}, + {"LUDS", "Volatility Trading Pause - Straddle Condition"}, + {"MWC0", "Market Wide Circuit Breaker Halt - Carry over from previous day"}, + {"MWC1", "Market Wide Circuit Breaker Halt - Level 1"}, + {"MWC2", "Market Wide Circuit Breaker Halt - Level 2"}, + {"MWC3", "Market Wide Circuit Breaker Halt - Level 3"}, + {"MWCQ", "Market Wide Circuit Breaker Resumption"}, + {"IPO1", "IPO Issue Not Yet Trading"}, + {"IPOQ", "IPO Security Released for Quotation (Nasdaq Securities Only)"}, + {"IPOE", "IPO Security - Positioning Window Extension (Nasdaq Securities Only)"}, + {"M1", "Corporate Action"}, + {"M2", "Quotation Not Available"}, + {"R1", "New Issue Available"}, + {"R2", "Issue Available"}, + {"R4", "Qualifications Issues Reviewed/Resolved; Quotations/Trading to Resume"}, + {"R9", "Filing Requirements Satisfied/Resolved; Quotations/Trading To Resume"}, + {"C3", "Issuer News Not Forthcoming; Quotations/Trading To Resume"}, + {"C4", "Qualifications Halt Ended; Maintenance Requirements Met; Resume"}, + {"C9", "Qualifications Halt Concluded; Filings Met; Quotes/Trades To Resume"}, + {"C11", "Trade Halt Concluded By Other Regulatory Authority; Quotes/Trades Resume"}, + {" ", "Reason Not Available"}, + }) }; -const std::map TRADING_STATES = { +inline constexpr FrozenMap TRADING_STATES {std::to_array>({ {'H', "Halted across all U.S. equity markets / SROs"}, {'P', "Paused across all U.S. equity markets / SROs"}, {'Q', "Quotation only period for cross-SRO halt or pause"}, {'T', "Trading on NASDAQ"}, -}; +})}; -const std::map MARKET_MAKER_MODE = { +inline constexpr FrozenMap MARKET_MAKER_MODE {std::to_array>({ {'N', "Normal"}, {'P', "Passive"}, {'S', "Syndicate"}, {'R', "Pre-syndicate"}, {'L', "Penalty"}, -}; +})}; -const std::map MARKET_PARTICIPANT_STATE = { - {'A', "Active"}, - {'E', "Excused"}, - {'W', "Withdrawn"}, - {'S', "Suspended"}, - {'D', "Deleted"}, +inline constexpr FrozenMap MARKET_PARTICIPANT_STATE { + std::to_array>({ + {'A', "Active"}, + {'E', "Excused"}, + {'W', "Withdrawn"}, + {'S', "Suspended"}, + {'D', "Deleted"}, + }) }; -const std::map PRICE_VARIATION_INDICATOR = { - {'L', "Less than 1%"}, - {'1', "1 to 1.99%"}, - {'2', "2 to 2.99%"}, - {'3', "3 to 3.99%"}, - {'4', "4 to 4.99%"}, - {'5', "5 to 5.99%"}, - {'6', "6 to 6.99%"}, - {'7', "7 to 7.99%"}, - {'8', "8 to 8.99%"}, - {'9', "9 to 9.99%"}, - {'A', "10 to 19.99%"}, - {'B', "20 to 29.99%"}, - {'C', "30% or greater"}, - {' ', "Cannot be calculated"}, +inline constexpr FrozenMap PRICE_VARIATION_INDICATOR { + std::to_array>({ + {'L', "Less than 1%"}, + {'1', "1 to 1.99%"}, + {'2', "2 to 2.99%"}, + {'3', "3 to 3.99%"}, + {'4', "4 to 4.99%"}, + {'5', "5 to 5.99%"}, + {'6', "6 to 6.99%"}, + {'7', "7 to 7.99%"}, + {'8', "8 to 8.99%"}, + {'9', "9 to 9.99%"}, + {'A', "10 to 19.99%"}, + {'B', "20 to 29.99%"}, + {'C', "30% or greater"}, + {' ', "Cannot be calculated"}, + }) }; } // namespace indicators From a73060b121c7f019367d4d3c7c7207e12a991179 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 14:59:16 +0100 Subject: [PATCH 007/144] Modernize includes and use pass-by-reference signatures Co-Authored-By: Claude Opus 4.8 --- include/itch/order_book.hpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/include/itch/order_book.hpp b/include/itch/order_book.hpp index 824d335..d0247d4 100644 --- a/include/itch/order_book.hpp +++ b/include/itch/order_book.hpp @@ -2,11 +2,10 @@ #include #include -#include -#include #include #include #include +#include #include #include @@ -53,11 +52,11 @@ struct PriceLevel { /// @brief Appends an order to the end of the queue (Time priority). /// @param order Shared pointer to the order to add. - auto add_order(std::shared_ptr order) -> void; + auto add_order(const std::shared_ptr& order) -> void; /// @brief Removes a specific order from the queue. /// @param order_it Iterator pointing to the order to remove. - auto remove_order(OrderIt order_it) -> void; + auto remove_order(const OrderIt& order_it) -> void; }; /// @class LimitOrderBook From f6ab8b27492d740a820936bacb717cc576d5e2fb Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 14:59:16 +0100 Subject: [PATCH 008/144] Render book with std::format and apply tidy cleanups Co-Authored-By: Claude Opus 4.8 --- src/order_book.cpp | 76 ++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/src/order_book.cpp b/src/order_book.cpp index f41a925..9e9d7be 100644 --- a/src/order_book.cpp +++ b/src/order_book.cpp @@ -1,18 +1,22 @@ #include "itch/order_book.hpp" #include +#include +#include #include #include +#include "itch/price.hpp" + namespace itch { -auto PriceLevel::add_order(std::shared_ptr order) -> void { +auto PriceLevel::add_order(const std::shared_ptr& order) -> void { total_shares += order->shares; orders.push_back(order); order->level = this; } -auto PriceLevel::remove_order(OrderIt order_it) -> void { orders.erase(order_it); } +auto PriceLevel::remove_order(const OrderIt& order_it) -> void { orders.erase(order_it); } auto LimitOrderBook::process(const Message& message) -> void { std::visit( @@ -27,44 +31,40 @@ auto LimitOrderBook::process(const Message& message) -> void { } auto LimitOrderBook::print(std::ostream& out, unsigned int delay_ms) const -> void { - std::ios state(nullptr); - state.copyfmt(out); - out << std::fixed << std::setprecision(4); + constexpr std::string_view BORDER = "==========================================\n"; + constexpr std::string_view MID_RULE = "-----------+--------------+--------------\n"; - out << "==========================================\n"; + out << BORDER; out << " SHARES | PRICE | SIDE \n"; - out << "==========================================\n"; + out << BORDER; - if (!m_asks.empty()) { - for (auto iter = m_asks.rbegin(); iter != m_asks.rend(); ++iter) { + const auto print_level = + [&](std::uint32_t total_shares, std::uint32_t raw_price, std::string_view side) { if (delay_ms > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); } - double price = iter->first / PRICE_DIVISOR; - out << std::setw(10) << iter->second.total_shares << " | " << std::setw(12) << price - << " | Ask\n"; - if (delay_ms > 0) + out << std::format( + "{:>10} | {:>12.4f} | {}\n", total_shares, make_price(raw_price).to_double(), side + ); + if (delay_ms > 0) { out << std::flush; - } + } + }; + + // Asks are stored low-to-high; print them high-to-low so the best ask sits + // just above the spread. + for (auto iter = m_asks.rbegin(); iter != m_asks.rend(); ++iter) { + print_level(iter->second.total_shares, iter->first, "Ask"); } - out << "-----------+--------------+--------------\n"; + out << MID_RULE; - if (!m_bids.empty()) { - for (const auto& [raw_price, level] : m_bids) { - if (delay_ms > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); - } - double price = raw_price / PRICE_DIVISOR; - out << std::setw(10) << level.total_shares << " | " << std::setw(12) << price - << " | Bid\n"; - if (delay_ms > 0) - out << std::flush; - } + // Bids are stored high-to-low, which is already the desired print order. + for (const auto& [raw_price, level] : m_bids) { + print_level(level.total_shares, raw_price, "Bid"); } - out << "==========================================\n"; - out.copyfmt(state); + out << BORDER; } auto LimitOrderBook::handle_message(const AddOrderMessage& msg) -> void { @@ -94,24 +94,24 @@ auto LimitOrderBook::handle_message(const AddOrderMPIDAttributionMessage& msg) - } auto LimitOrderBook::handle_message(const OrderExecutedMessage& msg) -> void { - auto it = m_orders.find(msg.order_reference_number); - if (it == m_orders.end() || (*it->second)->stock != m_stock_symbol) { + auto order_iter = m_orders.find(msg.order_reference_number); + if (order_iter == m_orders.end() || (*order_iter->second)->stock != m_stock_symbol) { return; } remove_order(msg.order_reference_number, msg.executed_shares); } auto LimitOrderBook::handle_message(const OrderExecutedWithPriceMessage& msg) -> void { - auto it = m_orders.find(msg.order_reference_number); - if (it == m_orders.end() || (*it->second)->stock != m_stock_symbol) { + auto order_iter = m_orders.find(msg.order_reference_number); + if (order_iter == m_orders.end() || (*order_iter->second)->stock != m_stock_symbol) { return; } remove_order(msg.order_reference_number, msg.executed_shares); } auto LimitOrderBook::handle_message(const OrderCancelMessage& msg) -> void { - auto it = m_orders.find(msg.order_reference_number); - if (it == m_orders.end() || (*it->second)->stock != m_stock_symbol) { + auto order_iter = m_orders.find(msg.order_reference_number); + if (order_iter == m_orders.end() || (*order_iter->second)->stock != m_stock_symbol) { return; } remove_order(msg.order_reference_number, msg.cancelled_shares); @@ -159,9 +159,11 @@ auto LimitOrderBook::remove_order(uint64_t order_ref, uint32_t canceled_shares) return; } - OrderIt order_iterator = iter->second; - auto order_ptr = *order_iterator; - PriceLevel* level = order_ptr->level; + auto order_iterator = iter->second; + // This copy is deliberate: it keeps the Order alive after the list node is + // erased below, since the order's price and side are read afterwards. + auto order_ptr = *order_iterator; // NOLINT(performance-unnecessary-copy-initialization) + PriceLevel* level = order_ptr->level; level->total_shares -= canceled_shares; order_ptr->shares -= canceled_shares; From 1417bc54b7aac35571a97c2616c0090e7ec31141 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:32 +0100 Subject: [PATCH 009/144] Add parser edge-case and error-policy tests Co-Authored-By: Claude Opus 4.8 --- tests/test_parser_edge.cpp | 216 +++++++++++++++++++++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 tests/test_parser_edge.cpp diff --git a/tests/test_parser_edge.cpp b/tests/test_parser_edge.cpp new file mode 100644 index 0000000..7aa4e8f --- /dev/null +++ b/tests/test_parser_edge.cpp @@ -0,0 +1,216 @@ +#include + +#include +#include +#include +#include + +#include "itch/parser.hpp" + +namespace { + +// Minimal big-endian wire-frame writer mirroring the ITCH layout the parser +// expects: a 2-byte big-endian length prefix followed by the payload, whose +// header is type(1) + stock_locate(2) + tracking_number(2) + 48-bit timestamp. +class FrameWriter { + public: + auto add_system_event( + std::uint16_t locate, std::uint16_t tracking, std::uint64_t timestamp, char event_code + ) -> void { + std::vector payload; + payload.push_back(static_cast('S')); + append_u16(payload, locate); + append_u16(payload, tracking); + append_u48(payload, timestamp); + payload.push_back(static_cast(event_code)); + append_frame(payload); + } + + auto add_order( + std::uint16_t locate, + std::uint64_t order_ref, + char side, + std::uint32_t shares, + const char (&stock)[9], + std::uint32_t price + ) -> void { + std::vector payload; + payload.push_back(static_cast('A')); + append_u16(payload, locate); + append_u16(payload, 0); + append_u48(payload, 0); + append_u64(payload, order_ref); + payload.push_back(static_cast(side)); + append_u32(payload, shares); + for (int idx = 0; idx < 8; ++idx) { + payload.push_back(static_cast(stock[idx])); + } + append_u32(payload, price); + append_frame(payload); + } + + // Writes a frame with a caller-chosen (possibly wrong) declared length. + auto add_raw_frame(std::uint16_t declared_length, const std::vector& payload) + -> void { + m_bytes.push_back(static_cast(declared_length >> 8)); + m_bytes.push_back(static_cast(declared_length & 0xFF)); + m_bytes.insert(m_bytes.end(), payload.begin(), payload.end()); + } + + [[nodiscard]] auto bytes() const -> const std::vector& { return m_bytes; } + + [[nodiscard]] auto as_byte_span() const -> std::span { + return std::as_bytes(std::span {m_bytes}); + } + + private: + static auto append_u16(std::vector& out, std::uint16_t value) -> void { + out.push_back(static_cast(value >> 8)); + out.push_back(static_cast(value & 0xFF)); + } + static auto append_u32(std::vector& out, std::uint32_t value) -> void { + for (int shift = 24; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + static auto append_u48(std::vector& out, std::uint64_t value) -> void { + for (int shift = 40; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + static auto append_u64(std::vector& out, std::uint64_t value) -> void { + for (int shift = 56; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + auto append_frame(const std::vector& payload) -> void { + add_raw_frame(static_cast(payload.size()), payload); + } + + std::vector m_bytes; +}; + +constexpr char STOCK_AAPL[9] = "AAPL "; + +} // namespace + +TEST(ParserEdge, EndiannessRoundTrip) { + EXPECT_EQ(itch::utils::swap_bytes(0x1234), 0x3412); + EXPECT_EQ(itch::utils::swap_bytes(0x11223344), 0x44332211u); + EXPECT_EQ(itch::utils::swap_bytes(0x0102030405060708ull), 0x0807060504030201ull); + + // Swapping twice is the identity. + const std::uint32_t value = 0xDEADBEEF; + EXPECT_EQ(itch::utils::swap_bytes(itch::utils::swap_bytes(value)), value); + + // A single byte is unchanged. + EXPECT_EQ(itch::utils::swap_bytes(0x7F), 0x7F); +} + +TEST(ParserEdge, SpanOverloadParsesIdenticallyToPointer) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + writer.add_order(1, 42, 'B', 100, STOCK_AAPL, 5000); + + itch::Parser parser; + auto via_span = parser.parse(writer.as_byte_span()); + ASSERT_EQ(via_span.size(), 2u); + EXPECT_TRUE(std::holds_alternative(via_span[0])); + EXPECT_TRUE(std::holds_alternative(via_span[1])); +} + +TEST(ParserEdge, ZeroLengthFramesAreSkipped) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + writer.add_raw_frame(0, {}); // Zero-length padding frame. + writer.add_system_event(4, 5, 6, 'C'); + + itch::Parser parser; + auto messages = parser.parse(writer.as_byte_span()); + EXPECT_EQ(messages.size(), 2u); +} + +TEST(ParserEdge, UnknownTypeIsSkippedAndCounted) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + // A well-formed-looking frame whose type byte '?' is not a known message. + writer.add_raw_frame(4, {static_cast('?'), 0, 0, 0}); + writer.add_system_event(4, 5, 6, 'C'); + + itch::Parser parser; + int seen = 0; + parser.parse(writer.as_byte_span(), [&](const itch::Message&) { ++seen; }); + + EXPECT_EQ(seen, 2); + EXPECT_EQ(parser.unknown_message_count(), 1u); + EXPECT_EQ(parser.malformed_message_count(), 0u); +} + +TEST(ParserEdge, UndersizedFrameIsRejectedNotDecodedIntoNextFrame) { + FrameWriter writer; + // A frame claiming type 'S' (needs 12 bytes) but only declaring 5: it must be + // rejected as malformed rather than reading into the following frame. + writer.add_raw_frame(5, {static_cast('S'), 0, 0, 0, 0}); + writer.add_system_event(7, 8, 9, 'C'); + + itch::Parser parser; + auto messages = parser.parse(writer.as_byte_span()); + + ASSERT_EQ(messages.size(), 1u); + ASSERT_TRUE(std::holds_alternative(messages[0])); + EXPECT_EQ(std::get(messages[0]).stock_locate, 7); + EXPECT_EQ(parser.malformed_message_count(), 1u); +} + +TEST(ParserEdge, ErrorCallbackReceivesEachProblem) { + FrameWriter writer; + writer.add_raw_frame(4, {static_cast('?'), 0, 0, 0}); // unknown + writer.add_raw_frame(5, {static_cast('S'), 0, 0, 0, 0}); // undersized + + itch::Parser parser; + std::vector errors; + parser.set_error_callback([&](itch::ParseError error, char) { errors.push_back(error); }); + parser.parse(writer.as_byte_span(), [](const itch::Message&) {}); + + ASSERT_EQ(errors.size(), 2u); + EXPECT_EQ(errors[0], itch::ParseError::unknown_type); + EXPECT_EQ(errors[1], itch::ParseError::size_mismatch); +} + +TEST(ParserEdge, ThrowsOnTruncatedPayload) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + // Declare a 12-byte frame but provide only 4 payload bytes. + writer.add_raw_frame(12, {static_cast('S'), 0, 0, 0}); + + itch::Parser parser; + EXPECT_THROW(parser.parse(writer.as_byte_span()), std::runtime_error); +} + +#if defined(__cpp_lib_expected) +TEST(ParserEdge, TryParseReturnsTruncatedInsteadOfThrowing) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + writer.add_raw_frame(12, {static_cast('S'), 0, 0, 0}); // truncated + + itch::Parser parser; + int seen = 0; + auto result = parser.try_parse(writer.as_byte_span(), [&](const itch::Message&) { ++seen; }); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), itch::ParseError::truncated); + EXPECT_EQ(seen, 1); // The first, complete frame was still delivered. +} + +TEST(ParserEdge, TryParseSucceedsOnWellFormedInput) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + writer.add_order(1, 42, 'B', 100, STOCK_AAPL, 5000); + + itch::Parser parser; + auto result = parser.try_parse(writer.as_byte_span()); + + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->size(), 2u); +} +#endif From ecd81239ac627a5b63c34d41f315204c2c27413a Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:32 +0100 Subject: [PATCH 010/144] Add Price and timestamp helper tests Co-Authored-By: Claude Opus 4.8 --- tests/test_price_time.cpp | 63 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 tests/test_price_time.cpp diff --git a/tests/test_price_time.cpp b/tests/test_price_time.cpp new file mode 100644 index 0000000..e8fe537 --- /dev/null +++ b/tests/test_price_time.cpp @@ -0,0 +1,63 @@ +#include + +#include +#include +#include + +#include "itch/price.hpp" +#include "itch/time.hpp" + +TEST(PriceTest, StandardScaleUsesFourDecimals) { + const auto price = itch::make_price(1500000); // 150.0000 + EXPECT_EQ(price.raw(), 1500000u); + EXPECT_EQ(itch::StandardPrice::divisor(), 10000u); + EXPECT_DOUBLE_EQ(price.to_double(), 150.0); + EXPECT_EQ(price.to_string(), "150.0000"); +} + +TEST(PriceTest, MwcbScaleUsesEightDecimals) { + const auto price = itch::make_mwcb_price(15000000000ull); // 150.00000000 + EXPECT_EQ(itch::MwcbPrice::divisor(), 100000000ull); + EXPECT_DOUBLE_EQ(price.to_double(), 150.0); + EXPECT_EQ(price.to_string(), "150.00000000"); +} + +TEST(PriceTest, TheTwoScalesAreDistinctTypes) { + // The standard and MWCB prices are different types, so the 4-vs-8 decimal + // mix-up cannot compile. This is a documentation-as-test of that intent. + static_assert(!std::is_same_v); + EXPECT_EQ(itch::StandardPrice::decimals, 4u); + EXPECT_EQ(itch::MwcbPrice::decimals, 8u); +} + +TEST(PriceTest, ComparesByRawValue) { + EXPECT_TRUE(itch::make_price(100) < itch::make_price(200)); + EXPECT_TRUE(itch::make_price(200) == itch::make_price(200)); +} + +TEST(PriceTest, FormatterRendersThroughStdFormat) { + EXPECT_EQ(std::format("{:.4f}", itch::make_price(1500000)), "150.0000"); +} + +TEST(TimeTest, CombinesSessionDateAndTimestamp) { + using namespace std::chrono; + // 09:30:00.000000000 = 34200 seconds past midnight. + constexpr std::uint64_t nanos = 34200ull * 1'000'000'000ull; + const auto point = itch::to_time_point(2020y / January / 30, nanos); + + const auto expected = sys_days {2020y / January / 30} + hours {9} + minutes {30}; + EXPECT_EQ(point, expected); +} + +TEST(TimeTest, FormatsTimestampAsTimeOfDay) { + constexpr std::uint64_t nanos = 34200ull * 1'000'000'000ull; + EXPECT_EQ(itch::format_timestamp(nanos), "09:30:00.000000000"); +} + +TEST(TimeTest, FormatsFullTimePoint) { + using namespace std::chrono; + constexpr std::uint64_t nanos = 34200ull * 1'000'000'000ull; + const auto text = itch::format_time_point(2020y / January / 30, nanos); + EXPECT_NE(text.find("2020-01-30"), std::string::npos); + EXPECT_NE(text.find("09:30:00"), std::string::npos); +} From 9fe41d34efa5b3967b5e17ca2d17fcfe28ca20b7 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:32 +0100 Subject: [PATCH 011/144] Add golden conformance test Co-Authored-By: Claude Opus 4.8 --- tests/test_conformance.cpp | 134 +++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 tests/test_conformance.cpp diff --git a/tests/test_conformance.cpp b/tests/test_conformance.cpp new file mode 100644 index 0000000..ae8f6cd --- /dev/null +++ b/tests/test_conformance.cpp @@ -0,0 +1,134 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include "itch/parser.hpp" + +// A golden/conformance test: a deterministic, hand-built ITCH stream with a known +// composition is parsed, and the per-type message counts and spot-checked field +// values are asserted against the known-good fixture. This gives an end-to-end +// "known input produces known output" check without committing a multi-gigabyte +// binary sample to the repository. +namespace { + +class StreamBuilder { + public: + auto system_event(char event_code) -> void { + std::vector payload; + payload.push_back(static_cast('S')); + header(payload, ++m_seq); + payload.push_back(static_cast(event_code)); + frame(payload); + } + + auto add_order(std::uint64_t order_ref, char side, std::uint32_t shares, std::uint32_t price) + -> void { + std::vector payload; + payload.push_back(static_cast('A')); + header(payload, ++m_seq); + u64(payload, order_ref); + payload.push_back(static_cast(side)); + u32(payload, shares); + for (int idx = 0; idx < 8; ++idx) { + payload.push_back(static_cast("AAPL "[idx])); + } + u32(payload, price); + frame(payload); + } + + auto order_executed(std::uint64_t order_ref, std::uint32_t executed, std::uint64_t match) + -> void { + std::vector payload; + payload.push_back(static_cast('E')); + header(payload, ++m_seq); + u64(payload, order_ref); + u32(payload, executed); + u64(payload, match); + frame(payload); + } + + [[nodiscard]] auto span() const -> std::span { + return std::as_bytes(std::span {m_bytes}); + } + + private: + static auto u16(std::vector& out, std::uint16_t value) -> void { + out.push_back(static_cast(value >> 8)); + out.push_back(static_cast(value & 0xFF)); + } + static auto u32(std::vector& out, std::uint32_t value) -> void { + for (int shift = 24; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + static auto u48(std::vector& out, std::uint64_t value) -> void { + for (int shift = 40; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + static auto u64(std::vector& out, std::uint64_t value) -> void { + for (int shift = 56; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + static auto header(std::vector& out, std::uint16_t seq) -> void { + u16(out, seq); // stock_locate + u16(out, 0); // tracking_number + u48(out, seq); // timestamp + } + auto frame(const std::vector& payload) -> void { + const auto length = static_cast(payload.size()); + m_bytes.push_back(static_cast(length >> 8)); + m_bytes.push_back(static_cast(length & 0xFF)); + m_bytes.insert(m_bytes.end(), payload.begin(), payload.end()); + } + + std::vector m_bytes; + std::uint16_t m_seq {0}; +}; + +} // namespace + +TEST(Conformance, KnownStreamProducesKnownCountsAndFields) { + StreamBuilder builder; + // Known composition: 3 system events, 5 add orders, 2 executions. + builder.system_event('O'); + builder.add_order(10, 'B', 100, 5000); + builder.add_order(11, 'B', 200, 4900); + builder.system_event('S'); + builder.add_order(12, 'S', 150, 5100); + builder.add_order(13, 'S', 50, 5200); + builder.order_executed(10, 40, 1); + builder.add_order(14, 'B', 300, 4800); + builder.order_executed(12, 150, 2); + builder.system_event('C'); + + itch::Parser parser; + auto messages = parser.parse(builder.span()); + + std::map counts; + for (const auto& message : messages) { + const char type = std::visit([](auto&& msg) { return msg.message_type; }, message); + ++counts[type]; + } + + EXPECT_EQ(messages.size(), 10u); + EXPECT_EQ(counts['S'], 3); + EXPECT_EQ(counts['A'], 5); + EXPECT_EQ(counts['E'], 2); + EXPECT_EQ(parser.unknown_message_count(), 0u); + EXPECT_EQ(parser.malformed_message_count(), 0u); + + // Spot-check fields of the first add order. + const auto& first_add = std::get(messages[1]); + EXPECT_EQ(first_add.order_reference_number, 10u); + EXPECT_EQ(first_add.buy_sell_indicator, 'B'); + EXPECT_EQ(first_add.shares, 100u); + EXPECT_EQ(first_add.price, 5000u); + EXPECT_EQ(itch::to_string(first_add.stock, 8), "AAPL"); +} From dcd7061d4ec98e950cc7f044974c597fc943c0a7 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:32 +0100 Subject: [PATCH 012/144] Add crossed-book and full-lifecycle book tests Co-Authored-By: Claude Opus 4.8 --- tests/test_order_book_extra.cpp | 98 +++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 tests/test_order_book_extra.cpp diff --git a/tests/test_order_book_extra.cpp b/tests/test_order_book_extra.cpp new file mode 100644 index 0000000..604fd7b --- /dev/null +++ b/tests/test_order_book_extra.cpp @@ -0,0 +1,98 @@ +#include + +#include + +#include "itch/messages.hpp" +#include "itch/order_book.hpp" + +namespace { + +auto make_add(std::uint64_t ref, char side, std::uint32_t shares, std::uint32_t price) + -> itch::AddOrderMessage { + itch::AddOrderMessage msg {}; + std::memcpy(msg.stock, "ZVZZT ", 8); + msg.order_reference_number = ref; + msg.buy_sell_indicator = side; + msg.shares = shares; + msg.price = price; + return msg; +} + +} // namespace + +class OrderBookExtra : public ::testing::Test { + protected: + OrderBookExtra() : book("ZVZZT") {} + itch::LimitOrderBook book; +}; + +// A crossed book (best bid >= best ask) can legitimately occur in a raw feed. +// The reconstruction must store both sides without matching or losing orders. +TEST_F(OrderBookExtra, CrossedBookKeepsBothSides) { + book.process(make_add(1, 'B', 100, 5100)); // Aggressive bid. + book.process(make_add(2, 'S', 100, 5000)); // Aggressive ask, below the bid. + + ASSERT_EQ(book.get_bids().size(), 1u); + ASSERT_EQ(book.get_asks().size(), 1u); + + const std::uint32_t best_bid = book.get_bids().begin()->first; + const std::uint32_t best_ask = book.get_asks().begin()->first; + EXPECT_EQ(best_bid, 5100u); + EXPECT_EQ(best_ask, 5000u); + EXPECT_GT(best_bid, best_ask); // The book is crossed, and that is preserved. +} + +TEST_F(OrderBookExtra, BestBidIsHighestBestAskIsLowest) { + book.process(make_add(1, 'B', 100, 4900)); + book.process(make_add(2, 'B', 100, 5000)); + book.process(make_add(3, 'B', 100, 4800)); + book.process(make_add(4, 'S', 100, 5300)); + book.process(make_add(5, 'S', 100, 5200)); + book.process(make_add(6, 'S', 100, 5400)); + + EXPECT_EQ(book.get_bids().begin()->first, 5000u); // Highest bid first. + EXPECT_EQ(book.get_asks().begin()->first, 5200u); // Lowest ask first. +} + +// Walk a single order through its full lifecycle across message types. +TEST_F(OrderBookExtra, FullLifecycleAddExecuteReplaceCancelDelete) { + book.process(make_add(100, 'B', 1000, 5000)); + ASSERT_EQ(book.get_bids().at(5000).total_shares, 1000u); + + // Partial execution. + itch::OrderExecutedMessage exec {}; + exec.order_reference_number = 100; + exec.executed_shares = 250; + book.process(exec); + EXPECT_EQ(book.get_bids().at(5000).total_shares, 750u); + + // Execution with price (partial). + itch::OrderExecutedWithPriceMessage exec_price {}; + exec_price.order_reference_number = 100; + exec_price.executed_shares = 250; + exec_price.execution_price = 5001; + book.process(exec_price); + EXPECT_EQ(book.get_bids().at(5000).total_shares, 500u); + + // Replace the remainder to a new price level. + itch::OrderReplaceMessage replace {}; + replace.original_order_reference_number = 100; + replace.new_order_reference_number = 101; + replace.shares = 500; + replace.price = 5050; + book.process(replace); + EXPECT_EQ(book.get_bids().count(5000), 0u); + EXPECT_EQ(book.get_bids().at(5050).total_shares, 500u); + + // Partial cancel, then delete the rest. + itch::OrderCancelMessage cancel {}; + cancel.order_reference_number = 101; + cancel.cancelled_shares = 200; + book.process(cancel); + EXPECT_EQ(book.get_bids().at(5050).total_shares, 300u); + + itch::OrderDeleteMessage del {}; + del.order_reference_number = 101; + book.process(del); + EXPECT_TRUE(book.get_bids().empty()); +} From 7fd38fde805cc575e77fd64462045e07ad5813b3 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:33 +0100 Subject: [PATCH 013/144] Register new test files Co-Authored-By: Claude Opus 4.8 --- tests/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d6bec98..95c9d30 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.23) +cmake_minimum_required(VERSION 3.25) find_package(GTest CONFIG QUIET) @@ -16,8 +16,12 @@ endif() add_executable( itch_tests test_parser.cpp + test_parser_edge.cpp test_messages.cpp test_order_book.cpp + test_order_book_extra.cpp + test_price_time.cpp + test_conformance.cpp ) target_link_libraries( From 20cffd49395eda952d513507adc27b4d9eed4446 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:33 +0100 Subject: [PATCH 014/144] Add libFuzzer parser target Co-Authored-By: Claude Opus 4.8 --- fuzz/CMakeLists.txt | 19 +++++++++++++++++++ fuzz/parser_fuzzer.cpp | 26 ++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 fuzz/CMakeLists.txt create mode 100644 fuzz/parser_fuzzer.cpp diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt new file mode 100644 index 0000000..2c1c9d0 --- /dev/null +++ b/fuzz/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.25) + +# libFuzzer ships with Clang. On other compilers the fuzz target is simply not +# created, so enabling the option elsewhere is a harmless no-op. +if (NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + message(STATUS "Fuzzers require Clang (libFuzzer); skipping fuzz targets.") + return() +endif() + +# Instrument the library itself so the fuzzer gets coverage feedback through it, +# while leaving the main (non-fuzz) builds of the library untouched. +target_compile_options(itch PRIVATE -fsanitize=fuzzer-no-link,address,undefined) + +add_executable(parser_fuzzer parser_fuzzer.cpp) +target_link_libraries(parser_fuzzer PRIVATE itch::itch) +target_compile_options(parser_fuzzer PRIVATE -g -fsanitize=fuzzer,address,undefined) +target_link_options(parser_fuzzer PRIVATE -fsanitize=fuzzer,address,undefined) + +message(STATUS "Added 'parser_fuzzer' target") diff --git a/fuzz/parser_fuzzer.cpp b/fuzz/parser_fuzzer.cpp new file mode 100644 index 0000000..daf62a2 --- /dev/null +++ b/fuzz/parser_fuzzer.cpp @@ -0,0 +1,26 @@ +#include +#include +#include +#include + +#include "itch/parser.hpp" + +// libFuzzer entry point. Feeds arbitrary bytes straight through the framing and +// dispatch path. Truncation is reported by `parse` as an exception, which is an +// expected outcome on random input and is swallowed; any crash, overread, or +// sanitizer report on this path is a genuine bug. +extern "C" auto LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size) -> int { + itch::Parser parser; + + // std::byte and the incoming bytes may legitimately alias; route through + // void* to avoid a forbidden reinterpret_cast. + const auto* bytes = static_cast(static_cast(data)); + const std::span buffer {bytes, size}; + + try { + parser.parse(buffer, [](const itch::Message&) noexcept {}); + } catch (const std::exception&) { + // Truncated input is expected and benign for the fuzzer. + } + return 0; +} From e91237a3a71430124e96874bcb1eb944445e00a5 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:33 +0100 Subject: [PATCH 015/144] Add ITCH sample download script Co-Authored-By: Claude Opus 4.8 --- scripts/fetch_sample.sh | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 scripts/fetch_sample.sh diff --git a/scripts/fetch_sample.sh b/scripts/fetch_sample.sh new file mode 100644 index 0000000..ead1da8 --- /dev/null +++ b/scripts/fetch_sample.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Downloads an official NASDAQ TotalView-ITCH 5.0 sample file for use by the +# benchmarks and the golden/conformance tests. The raw sample is a multi- +# gigabyte binary and is deliberately NOT committed to the repository: fetch it +# on demand instead. +# +# Usage: +# scripts/fetch_sample.sh [destination_dir] +# +# Environment: +# ITCH_SAMPLE_URL Override the download URL (defaults to the NASDAQ archive). +# +# The default URL points at NASDAQ's public sample archive. NASDAQ occasionally +# changes the available dates; if the default 404s, browse the index and set +# ITCH_SAMPLE_URL to a current file: +# https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/ + +set -euo pipefail + +readonly DEFAULT_URL="https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/01302020.NASDAQ_ITCH50.gz" +readonly SAMPLE_URL="${ITCH_SAMPLE_URL:-${DEFAULT_URL}}" + +dest_dir="${1:-data}" +mkdir -p "${dest_dir}" + +archive_name="$(basename "${SAMPLE_URL}")" +archive_path="${dest_dir}/${archive_name}" +decompressed_path="${archive_path%.gz}" + +if [[ -f "${decompressed_path}" ]]; then + echo "Sample already present: ${decompressed_path}" + exit 0 +fi + +echo "Downloading ITCH sample from: ${SAMPLE_URL}" +if command -v curl >/dev/null 2>&1; then + curl -fL --retry 3 -o "${archive_path}" "${SAMPLE_URL}" +elif command -v wget >/dev/null 2>&1; then + wget -O "${archive_path}" "${SAMPLE_URL}" +else + echo "error: neither curl nor wget is available" >&2 + exit 1 +fi + +if [[ "${archive_path}" == *.gz ]]; then + echo "Decompressing ${archive_path}" + gzip -d -k "${archive_path}" +fi + +echo "Sample ready: ${decompressed_path}" From 0b5ad19888f39d9bca79babe4b4fcd463ff37935 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:59 +0100 Subject: [PATCH 016/144] Fix helper names, bump CMake to 3.25, add fuzz target Co-Authored-By: Claude Opus 4.8 --- CMakeLists.txt | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0981ef1..2b27e6f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,9 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) file(READ "VERSION.txt" RAW_VERSION) string(STRIP "${RAW_VERSION}" APP_VERSION) -project(ITCH VERSION ${APP_VERSION}) +project(ITCH LANGUAGES CXX VERSION ${APP_VERSION}) include(${CMAKE_SOURCE_DIR}/cmake/Helpers.cmake) include(${CMAKE_SOURCE_DIR}/cmake/Versions.cmake) @@ -12,16 +12,16 @@ if(${PROJECT_NAME}_ADD_COVERAGE_ANALYSIS) add_coverage_analysis() endif() if(${PROJECT_NAME}_APPLY_FORMATING) - apply_formating(${PROJECT_NAME}_USE_PER_FILE_FORMATTING) + apply_formatting(${PROJECT_NAME}_USE_PER_FILE_FORMATTING) endif() if(${PROJECT_NAME}_APPLY_CLANG_TIDY_GLOBALY) - apply_clang_tidy_globaly() + apply_clang_tidy_globally() endif() if(${PROJECT_NAME}_BUILD_DOCUMENTATION) - build_documenation() + build_documentation() endif() if(${PROJECT_NAME}_ENABLE_ADDRESS_SANITIZER) - anable_address_sanitizer() + enable_address_sanitizer() endif() add_subdirectory(src) @@ -36,5 +36,8 @@ endif() if (${PROJECT_NAME}_BUILD_EXAMPLES) add_subdirectory(examples) endif() +if (${PROJECT_NAME}_BUILD_FUZZERS) + add_subdirectory(fuzz) +endif() From f5648972e300f59d61850aa776b8c6d2240b0d2e Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:59 +0100 Subject: [PATCH 017/144] Fix helper name typos, bump CMake min, add fuzz option Co-Authored-By: Claude Opus 4.8 --- cmake/Helpers.cmake | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cmake/Helpers.cmake b/cmake/Helpers.cmake index 8af1937..d3b4def 100644 --- a/cmake/Helpers.cmake +++ b/cmake/Helpers.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.23) +cmake_minimum_required(VERSION 3.25) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -8,16 +8,17 @@ set(CMAKE_CXX_STANDARD ${${PROJECT_NAME}_CXX_STANDARD}) option(${PROJECT_NAME}_BUILD_TESTS "Add tests" OFF) option(${PROJECT_NAME}_BUILD_BENCHMARKS "Add benchmark analysis" OFF) option(${PROJECT_NAME}_BUILD_EXAMPLES "Build some examples" OFF) +option(${PROJECT_NAME}_BUILD_FUZZERS "Build libFuzzer targets (Clang only)" OFF) set(${PROJECT_NAME}_PROJECT_ENV "DEV" CACHE STRING "Development environment") set_property(CACHE ${PROJECT_NAME}_PROJECT_ENV PROPERTY STRINGS "DEV" "PROD") message(STATUS "Building for environment: ${${PROJECT_NAME}_PROJECT_ENV}") -option(${PROJECT_NAME}_ADD_COVERAGE_ANALYSIS "Enable coverage analisys" OFF) -option(${PROJECT_NAME}_APPLY_FORMATING "Apply formating with clang-format" OFF) +option(${PROJECT_NAME}_ADD_COVERAGE_ANALYSIS "Enable coverage analysis" OFF) +option(${PROJECT_NAME}_APPLY_FORMATING "Apply formatting with clang-format" OFF) option(${PROJECT_NAME}_USE_PER_FILE_FORMATTING "For very large projects" OFF) -option(${PROJECT_NAME}_APPLY_CLANG_TIDY_GLOBALY "Apply clang tidy globaly" OFF) -option(${PROJECT_NAME}_BUILD_DOCUMENTATION "Build documenation with Doxygen" OFF) +option(${PROJECT_NAME}_APPLY_CLANG_TIDY_GLOBALY "Apply clang tidy globally" OFF) +option(${PROJECT_NAME}_BUILD_DOCUMENTATION "Build documentation with Doxygen" OFF) option(${PROJECT_NAME}_ENABLE_ADDRESS_SANITIZER "Enable Address Sanitizer" OFF) ###################################################### @@ -86,7 +87,7 @@ function(apply_clang_tidy TARGET_NAME) endfunction() ################################################### -function(apply_clang_tidy_globaly) +function(apply_clang_tidy_globally) find_program(CLANG_TIDY_EXE "clang-tidy") if(CLANG_TIDY_EXE) @@ -133,7 +134,7 @@ function(add_coverage_analysis) endfunction() #################################################### -function(build_documenation) +function(build_documentation) find_package(Doxygen) if (Doxygen_FOUND) message( @@ -156,7 +157,7 @@ function(build_documenation) endfunction() ##################################################### -function(apply_formating USE_PER_FILE_LOGIC) +function(apply_formatting USE_PER_FILE_LOGIC) find_program(CLANG_FORMAT_EXE clang-format) if (NOT CLANG_FORMAT_EXE) @@ -203,7 +204,7 @@ function(apply_formating USE_PER_FILE_LOGIC) endfunction() ##################################################### -function(anable_address_sanitizer) +function(enable_address_sanitizer) set(SANITIZER_SUPPORTED OFF) if (MINGW) if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") From a18691786c16898686402b23183b020d6e6d13b0 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:59 +0100 Subject: [PATCH 018/144] Bump CMake minimum to 3.25 Co-Authored-By: Claude Opus 4.8 --- cmake/Versions.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/Versions.cmake b/cmake/Versions.cmake index 7222516..0eff013 100644 --- a/cmake/Versions.cmake +++ b/cmake/Versions.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) set(GTEST_VERSION 1.17.0 CACHE STRING "") set(BENCHMARK_VERSION 1.9.4 CACHE STRING "") From 507cc2474a07091b5d75745d12a0f04808430863 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:59 +0100 Subject: [PATCH 019/144] Bump CMake minimum to 3.25 Co-Authored-By: Claude Opus 4.8 --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fe22a2b..d5da1a3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) include(GNUInstallDirs) From 4d341115dfc6e56fdbe6bc3fdb120ffbc2ceef65 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:00:59 +0100 Subject: [PATCH 020/144] Bump CMake minimum to 3.25 Co-Authored-By: Claude Opus 4.8 --- benchmarks/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 2007ce9..061d816 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.23) +cmake_minimum_required(VERSION 3.25) find_package(benchmark CONFIG QUIET) From 10f888fe00565e582e75debd03a00cd6dfd0b478 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:01:00 +0100 Subject: [PATCH 021/144] Bump CMake minimum to 3.25 Co-Authored-By: Claude Opus 4.8 --- examples/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index bd554a8..a67a1fd 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) add_executable(parser_example parser/parser_example.cpp) From 08a0169c94634a68091bb9293826b2b79fca063b Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:01:00 +0100 Subject: [PATCH 022/144] Run CI on PRs with lint, sanitizer, fuzz, and bench jobs Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build.yml | 234 +++++++++++++++++++++++++++++++++++- 1 file changed, 232 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c098c5c..6cd3d9e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,8 @@ name: build on: push: branches: [ "main" ] + pull_request: + branches: [ "main" ] jobs: build-and-test: @@ -42,7 +44,7 @@ jobs: key: ${{ runner.os }}-${{ matrix.build_type }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} restore-keys: | ${{ runner.os }}-${{ matrix.build_type }}-cmake- - + - name: Set VCPKG Triplet shell: bash run: | @@ -72,7 +74,235 @@ jobs: - name: Build shell: bash run: cmake --build $BUILD_DIR --config ${{ matrix.build_type }} - + - name: Run tests shell: bash run: ctest --test-dir $BUILD_DIR --output-on-failure --build-config ${{ matrix.build_type }} + + format-check: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Install clang-format + run: sudo apt-get update && sudo apt-get install -y clang-format + + - name: Check formatting + run: | + find src include tests benchmarks examples -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.h' \) \ + -print0 | xargs -0 clang-format --dry-run --Werror + + clang-tidy: + runs-on: ubuntu-latest + env: + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Install clang-tidy + run: sudo apt-get update && sudo apt-get install -y clang-tidy + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + ${{ env.VCPKG_ROOT }} + ~/.cache/vcpkg + key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg- + + - name: Set up vcpkg + uses: lukka/run-vcpkg@v11 + + - name: Configure CMake (export compile commands) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_TESTS=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-linux + + - name: Run clang-tidy + run: | + find src -name '*.cpp' -print0 \ + | xargs -0 clang-tidy -p "$BUILD_DIR" + + sanitizers: + runs-on: ubuntu-latest + env: + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + ${{ env.VCPKG_ROOT }} + ~/.cache/vcpkg + key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg- + + - name: Set up vcpkg + uses: lukka/run-vcpkg@v11 + + - name: Configure CMake (ASAN + UBSAN) + # The ITCH message structs are deliberately `#pragma pack(1)` to match the + # wire layout, so their multi-byte members are intentionally unaligned. + # Unaligned loads are safe on the supported architectures, so the UBSAN + # alignment check is excluded; every other UBSAN check remains fatal. + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_TESTS=ON \ + -DITCH_ENABLE_ADDRESS_SANITIZER=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS="-fsanitize=undefined -fno-sanitize=alignment -fno-sanitize-recover=undefined" \ + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=undefined" \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-linux + + - name: Build + run: cmake --build $BUILD_DIR + + - name: Run tests under sanitizers + run: ctest --test-dir $BUILD_DIR --output-on-failure + + coverage: + runs-on: ubuntu-latest + env: + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Install gcovr + run: sudo apt-get update && sudo apt-get install -y gcovr + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + ${{ env.VCPKG_ROOT }} + ~/.cache/vcpkg + key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg- + + - name: Set up vcpkg + uses: lukka/run-vcpkg@v11 + + - name: Configure CMake (coverage) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_TESTS=ON \ + -DITCH_ADD_COVERAGE_ANALYSIS=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-linux + + - name: Build + run: cmake --build $BUILD_DIR + + - name: Run tests + run: ctest --test-dir $BUILD_DIR --output-on-failure + + - name: Generate coverage report + run: cmake --build $BUILD_DIR --target coverage + + - name: Upload coverage report + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: ${{ env.BUILD_DIR }}/coverage.html + if-no-files-found: ignore + + fuzz: + runs-on: ubuntu-latest + env: + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Install clang + run: sudo apt-get update && sudo apt-get install -y clang + + - name: Configure CMake (fuzzers) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_FUZZERS=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ + + - name: Build fuzzer + run: cmake --build $BUILD_DIR --target parser_fuzzer + + - name: Run fuzzer (time-budgeted) + run: $BUILD_DIR/fuzz/parser_fuzzer -max_total_time=60 -max_len=512 -print_final_stats=1 + + benchmark-smoke: + runs-on: ubuntu-latest + env: + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + ${{ env.VCPKG_ROOT }} + ~/.cache/vcpkg + key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg- + + - name: Set up vcpkg + uses: lukka/run-vcpkg@v11 + + - name: Configure CMake (benchmarks) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_BENCHMARKS=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-linux + + - name: Build benchmark + run: cmake --build $BUILD_DIR --target parser_bench + + - name: Generate a small synthetic ITCH sample + run: | + python3 - <<'PY' + frame = (bytes([0x00, 0x0C]) + b'S' + + (1).to_bytes(2, 'big') + (2).to_bytes(2, 'big') + + (3).to_bytes(6, 'big') + b'O') + with open('sample.itch', 'wb') as out: + out.write(frame * 1_000_000) + PY + + - name: Run benchmark (smoke) + run: $BUILD_DIR/benchmarks/parser_bench sample.itch --benchmark_min_time=1x From 9d5b68006ebd25b84d65ff61f5068dc3d80c9368 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:01:00 +0100 Subject: [PATCH 023/144] Disable noisy clang-tidy checks Co-Authored-By: Claude Opus 4.8 --- .clang-tidy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.clang-tidy b/.clang-tidy index 92c0be5..444c6a8 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -22,12 +22,16 @@ Checks: > google-*, -cppcoreguidelines-pro-bounds-pointer-arithmetic, -cppcoreguidelines-pro-bounds-array-to-pointer-decay, + -cppcoreguidelines-pro-bounds-constant-array-index, + -cppcoreguidelines-pro-bounds-avoid-unchecked-container-access, -cppcoreguidelines-owning-memory, -hicpp-signed-bitwise, -modernize-use-trailing-return-type, -modernize-use-nodiscard, -readability-identifier-naming, -readability-magic-numbers, + -cppcoreguidelines-avoid-magic-numbers, + -bugprone-easily-swappable-parameters, -google-build-using-namespace, -google-runtime-references, -cert-err58-cpp From 92beed1caa5ca3368ca4ccee0d43e4f11fe6ccb2 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:01:00 +0100 Subject: [PATCH 024/144] Ignore generated coverage report Co-Authored-By: Claude Opus 4.8 --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 5abe957..b07f192 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ scripts.sh *Presets.json .cache/ /packaging + +# Generated coverage reports must not be committed; they belong in the build tree. +coverage.html +docs/coverage.html From 2530ada052e98944a4bddd0c37a035e1e98b5b61 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 15:01:00 +0100 Subject: [PATCH 025/144] Correct performance claims and publish benchmark results Co-Authored-By: Claude Opus 4.8 --- README.md | 242 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 135 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index 7ccad7c..2e19d2e 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,12 @@ - # High-Performance NASDAQ ITCH 5.0 Parser [![Build](https://github.com/bbalouki/itchcpp/actions/workflows/build.yml/badge.svg)](https://github.com/bbalouki/itchcpp/actions/workflows/build.yml) ![Linux](https://img.shields.io/badge/Linux-FCC624?logo=linux&logoColor=black) ![macOS](https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=white) ![Windows](https://img.shields.io/badge/Windows-0078D6?logo=windows&logoColor=white) +![Vcpkg Version](https://img.shields.io/vcpkg/v/bbalouki-itch) [![C++20](https://img.shields.io/badge/C++-20-blue.svg)](https://isocpp.org/std/the-standard) -[![CMake](https://img.shields.io/badge/CMake-3.20+-blue.svg)](https://cmake.org/) +[![CMake](https://img.shields.io/badge/CMake-3.25+-blue.svg)](https://cmake.org/) [![CodeFactor](https://www.codefactor.io/repository/github/bbalouki/itchcpp/badge)](https://www.codefactor.io/repository/github/bbalouki/itchcpp) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![LinkedIn](https://img.shields.io/badge/LinkedIn-grey?logo=Linkedin&logoColor=white)](https://www.linkedin.com/in/bertin-balouki-simyeli-15b17a1a6/) @@ -21,25 +21,25 @@ A modern, high-performance C++20 library for parsing NASDAQ TotalView-ITCH 5.0 p 1. [Project Philosophy](#project-philosophy) 2. [Key Features](#key-features) 3. [Architectural Overview](#architectural-overview) - * [Zero-Overhead Deserialization](#zero-overhead-deserialization) - * [Type-Safe Message Handling](#type-safe-message-handling) - * [Endianness and Byte Order](#endianness-and-byte-order) + - [Fast, Allocation-Free Single-Pass Decoding](#fast-allocation-free-single-pass-decoding) + - [Type-Safe Message Handling](#type-safe-message-handling) + - [Endianness and Byte Order](#endianness-and-byte-order) 4. [Getting Started](#getting-started) - * [Prerequisites](#prerequisites) - * [Building the Project](#building-the-project) - * [Running Tests and Benchmarks](#running-tests-and-benchmarks) + - [Prerequisites](#prerequisites) + - [Building the Project](#building-the-project) + - [Running Tests and Benchmarks](#running-tests-and-benchmarks) 5. [Dependency Management](#dependency-management) 6. [Comprehensive Usage Guide](#comprehensive-usage-guide) - * [Example 1: Parsing a File into a Vector](#example-1-parsing-a-file-into-a-vector) - * [Example 2: High-Performance Callback-Based Parsing](#example-2-high-performance-callback-based-parsing) - * [Example 3: Filtering Messages by Type](#example-3-filtering-messages-by-type) - * [Example 4: A Complete, Compilable Example](#example-4-a-complete-compilable-example) + - [Example 1: Parsing a File into a Vector](#example-1-parsing-a-file-into-a-vector) + - [Example 2: High-Performance Callback-Based Parsing](#example-2-high-performance-callback-based-parsing) + - [Example 3: Filtering Messages by Type](#example-3-filtering-messages-by-type) + - [Example 4: A Complete, Compilable Example](#example-4-a-complete-compilable-example) 7. [API Reference: ITCH 5.0 Message Types](#api-reference-itch-50-message-types) - * [System Event Messages](#system-event-messages) - * [Stock and Administrative Messages](#stock-and-administrative-messages) - * [Order Messages](#order-messages) - * [Trade Messages](#trade-messages) - * [Imbalance and Price Discovery Messages](#imbalance-and-price-discovery-messages) + - [System Event Messages](#system-event-messages) + - [Stock and Administrative Messages](#stock-and-administrative-messages) + - [Order Messages](#order-messages) + - [Trade Messages](#trade-messages) + - [Imbalance and Price Discovery Messages](#imbalance-and-price-discovery-messages) 8. [Performance](#performance) 9. [Coding Standards](#coding-standards) 10. [Frequently Asked Questions (FAQ)](#frequently-asked-questions-faq) @@ -53,7 +53,7 @@ A modern, high-performance C++20 library for parsing NASDAQ TotalView-ITCH 5.0 p The design of this ITCH parser is guided by three principles: -1. **Performance Above All**: In market data processing, performance is not just a feature; it's a requirement. This library is architected to eliminate unnecessary overhead. It avoids dynamic memory allocations, data copies, and stream I/O in its critical path. The goal is to convert raw binary data into a usable C++ struct as efficiently as possible. +1. **Performance Above All**: In market data processing, performance is not just a feature; it's a requirement. This library is architected to eliminate unnecessary overhead. It avoids dynamic memory allocations and stream I/O in its critical path, decoding each frame in a single forward pass. The goal is to convert raw binary data into a usable C++ struct as efficiently as possible. 2. **Correctness and Safety**: Financial data is unforgiving. A single misinterpreted byte can corrupt analysis. We prioritize correctness by adhering strictly to the ITCH 5.0 specification and ensuring data is represented in a type-safe manner. @@ -63,16 +63,16 @@ The design of this ITCH parser is guided by three principles: ## Key Features -* **High Throughput**: Capable of multi-gigabyte-per-second parsing speeds on modern hardware. -* **Zero-Allocation Core**: The primary parsing loop performs zero dynamic memory allocations, minimizing latency and jitter. -* **Type-Safe API**: All ITCH messages are deserialized into a `std::variant` of dedicated, packed `struct`s, ensuring compile-time safety. -* **Specification Compliant**: Faithfully implements the message formats outlined in the Nasdaq TotalView-ITCH 5.0 specification. -* **Flexible Parsing Strategies**: - * Eagerly parse a buffer into a `std::vector` for convenience. - * Use a callback for efficient, low-memory stream processing of large files. - * Filter messages by type at the parsing stage to reduce processing load. -* **Cross-Platform**: Compatible with any platform supporting a C++20 compiler and CMake. -* **Minimal Dependencies**: Zero runtime dependencies. Test and benchmark libraries are managed by `vcpkg`. +- **High Throughput**: Multi-gigabyte-per-second parsing on modern hardware (see [Benchmarks](#benchmarks) for measured numbers). +- **Allocation-Free Core**: The callback-based parsing loop performs zero dynamic memory allocations, minimizing latency and jitter. +- **Type-Safe API**: All ITCH messages are deserialized into a `std::variant` of dedicated, packed `struct`s, ensuring compile-time safety. +- **Specification Compliant**: Faithfully implements the message formats outlined in the Nasdaq TotalView-ITCH 5.0 specification. +- **Flexible Parsing Strategies**: + - Eagerly parse a buffer into a `std::vector` for convenience. + - Use a callback for efficient, low-memory stream processing of large files. + - Filter messages by type at the parsing stage to reduce processing load. +- **Cross-Platform**: Compatible with any platform supporting a C++20 compiler and CMake. +- **Minimal Dependencies**: Zero runtime dependencies. Test and benchmark libraries are managed by `vcpkg`. --- @@ -80,17 +80,21 @@ The design of this ITCH parser is guided by three principles: The library is built on some key principles to deliver maximum performance and safety. -### Zero-Overhead Deserialization +### Fast, Allocation-Free Single-Pass Decoding + +The parser is a fast, allocation-free, single-pass field decoder. Each message type is defined as a packed `struct` whose layout mirrors the ITCH wire format, and each frame is decoded in one forward pass: fields are copied out and converted from network byte order directly into the typed `struct`, with no intermediate buffers and no per-message heap allocation on the callback path. -To achieve its performance, the parser uses a zero-overhead deserialization strategy. The parser achieves its speed by reading messages directly from the raw data stream without performing heavy data conversions. Each message type is defined in a compact format that matches exactly how the data appears on the network. This means the parser can quickly copy incoming bytes into a message structure in memory, skipping the usual per-field decoding steps. The result is near-zero overhead when reading and processing messages. +Dispatch is driven by a compile-time table keyed on the one-byte message type, so selecting the right decoder is a single flat-array lookup followed by a direct, inlinable call. There is no red-black-tree lookup and no type-erased `std::function` indirection. The frame length declared on the wire is validated against the size the message type requires before any field is read, so a truncated or malformed frame can never cause the decoder to read into an adjacent message. + +This is genuine single-pass decoding rather than a zero-copy overlay: the bytes are converted eagerly into host-order fields. A lazy zero-copy view API (typed overlays over the raw buffer with on-access endianness conversion) is tracked as future work in [FEATURES.md](FEATURES.md). ### Type-Safe Message Handling All possible ITCH messages are handled using a single, modern C++ object: `itch::Message`, which is a `std::variant`. This approach provides significant advantages over traditional designs: -* **Compile-Time Safety:** The compiler verifies that your code handles every possible message type. This prevents unexpected errors at runtime if a message type is overlooked. -* **High Performance:** This design allows the compiler to generate highly optimized code for processing different messages, which is typically faster than conventional virtual function lookups. -* **Efficient Memory Layout:** Messages are stored contiguously in memory, which improves CPU cache utilization and overall processing speed. +- **Compile-Time Safety:** The compiler verifies that your code handles every possible message type. This prevents unexpected errors at runtime if a message type is overlooked. +- **High Performance:** This design allows the compiler to generate highly optimized code for processing different messages, which is typically faster than conventional virtual function lookups. +- **Efficient Memory Layout:** Messages are stored contiguously in memory, which improves CPU cache utilization and overall processing speed. ### Endianness and Byte Order @@ -104,12 +108,13 @@ The parser handles this discrepancy automatically. After a message is read into ### Prerequisites -* **C++ Compiler**: A compiler with full C++20 support (e.g., GCC 10+, Clang 12+, MSVC 19.29+). -* **CMake**: Version 3.20 or newer. +- **C++ Compiler**: A compiler with full C++20 support (e.g., GCC 10+, Clang 12+, MSVC 19.29+). +- **CMake**: Version 3.25 or newer. ### Building the Project 1. **Clone the Repository** + ```bash git clone https://github.com/bbalouki/itchcpp.git cd itchcpp @@ -117,6 +122,7 @@ The parser handles this discrepancy automatically. After a message is read into 2. **Configure with CMake** This step generates the build system and automatically downloads any developer dependencies if needed. + ```bash cmake -S . -B build ``` @@ -131,29 +137,31 @@ The parser handles this discrepancy automatically. After a message is read into The project includes unit tests and performance benchmarks that can be built by enabling the corresponding CMake options. -* **To build and run tests:** - ```bash - # Configure with tests enabled - cmake -S . -B build -DITCH_BUILD_TESTS=ON +- **To build and run tests:** - # Build - cmake --build build --config Release + ```bash + # Configure with tests enabled + cmake -S . -B build -DITCH_BUILD_TESTS=ON - # Run tests from the build directory - ctest --test-dir build --output-on-failure - ``` + # Build + cmake --build build --config Release -* **To build and run benchmarks:** - ```bash - # Configure with benchmarks enabled - cmake -S . -B build -DITCH_BUILD_BENCHMARKS=ON + # Run tests from the build directory + ctest --test-dir build --output-on-failure + ``` - # Build - cmake --build build --config Release +- **To build and run benchmarks:** - # Run benchmarks - ./build/benchmarks/parser_bench - ``` + ```bash + # Configure with benchmarks enabled + cmake -S . -B build -DITCH_BUILD_BENCHMARKS=ON + + # Build + cmake --build build --config Release + + # Run benchmarks + ./build/benchmarks/parser_bench + ``` --- @@ -162,8 +170,9 @@ The project includes unit tests and performance benchmarks that can be built by This library has **zero external runtime dependencies**. For development purposes, it uses two popular libraries: -* **Google Test**: For unit testing. -* **Google Benchmark**: For performance microbenchmarks. + +- **Google Test**: For unit testing. +- **Google Benchmark**: For performance microbenchmarks. These are handled automatically by Microsoft's `vcpkg` package manager. When you enable tests or benchmarks, **You will need to install them on your system separately or install them in manifest mode.** @@ -311,20 +320,20 @@ This standalone example demonstrates how to use `std::visit` with a visitor to p // Using `if constexpr` ensures only valid code is compiled for each message type. auto message_visitor = [](const auto& msg) { using T = std::decay_t; - + // Set fixed-point notation for prices std::cout << std::fixed << std::setprecision(4); if constexpr (std::is_same_v) { std::cout << "ADD ORDER: Ref #" << msg.order_reference_number - << ", " << msg.shares << " shares of " + << ", " << msg.shares << " shares of " << itch::to_string(msg.stock, sizeof(msg.stock)) // Helper to convert char array << " @ $" << static_cast(msg.price) / itch::PRICE_DIVISOR << "\n"; } else if constexpr (std::is_same_v) { std::cout << "EXECUTION: Ref #" << msg.order_reference_number << ", executed " << msg.executed_shares << " shares.\n"; } else if constexpr (std::is_same_v) { - std::cout << "SYSTEM EVENT: Code '" << msg.event_code + std::cout << "SYSTEM EVENT: Code '" << msg.event_code << "' (O=Start, S=SysHours, Q=MktHours, M=EndMkt, E=EndSys, C=End)\n"; } else { // This block will be visited for any other message types. @@ -388,7 +397,7 @@ int main(int argc, char* argv[]) { // Create a parser and configure it to only process book-related messages // This is a performance optimization. itch::Parser parser; - std::vector book_messages(order_book.book_messages.begin(), + std::vector book_messages(order_book.book_messages.begin(), order_book.book_messages.end()); try { @@ -421,9 +430,9 @@ The `print()` method provides a formatted snapshot of the top of the book. It di +-----------------------------------------------------+ | AAPL | +--------------------------+--------------------------+ -| ASKS (SELL) | BIDS (BUY) | +| ASKS (SELL) | BIDS (BUY) | +------------+-------------+------------+-------------+ -| PRICE | VOLUME | PRICE | VOLUME | +| PRICE | VOLUME | PRICE | VOLUME | +------------+-------------+------------+-------------+ | 173.5000 | 15,000 | 173.4900 | 23,500 | | 173.5100 | 12,300 | 173.4800 | 18,900 | @@ -441,54 +450,51 @@ This library supports all message types defined in the ITCH 5.0 specification. T ### System Event Messages -| Type | Struct Name | Description | -| :--: | -------------------- | ---------------------------------------------- | -| `S` | `SystemEventMessage` | Signals a market or data feed handler event. | +| Type | Struct Name | Description | +| :--: | -------------------- | -------------------------------------------- | +| `S` | `SystemEventMessage` | Signals a market or data feed handler event. | ### Stock and Administrative Messages -| Type | Struct Name | Description | -| :--: | ---------------------------------- | ------------------------------------------------------------------------- | -| `R` | `StockDirectoryMessage` | Conveys stock symbol directory information. | -| `H` | `StockTradingActionMessage` | Indicates a change in trading status for a security (e.g., Halted, Trading).| -| `Y` | `RegSHORestrictionMessage` | Regulation SHO short sale price test restricted indicator. | -| `L` | `MarketParticipantPositionMessage` | Conveys a market participant's status (e.g., Primary Market Maker). | -| `V` | `MWCBDeclineLevelMessage` | Informs of the Market-Wide Circuit Breaker (MWCB) breach points for the day.| -| `W` | `MWCBStatusMessage` | Indicates that a MWCB level has been breached. | -| `h` | `OperationalHaltMessage` | Indicates an operational halt for a security on a specific market center. | - +| Type | Struct Name | Description | +| :--: | ---------------------------------- | ---------------------------------------------------------------------------- | +| `R` | `StockDirectoryMessage` | Conveys stock symbol directory information. | +| `H` | `StockTradingActionMessage` | Indicates a change in trading status for a security (e.g., Halted, Trading). | +| `Y` | `RegSHORestrictionMessage` | Regulation SHO short sale price test restricted indicator. | +| `L` | `MarketParticipantPositionMessage` | Conveys a market participant's status (e.g., Primary Market Maker). | +| `V` | `MWCBDeclineLevelMessage` | Informs of the Market-Wide Circuit Breaker (MWCB) breach points for the day. | +| `W` | `MWCBStatusMessage` | Indicates that a MWCB level has been breached. | +| `h` | `OperationalHaltMessage` | Indicates an operational halt for a security on a specific market center. | ### Order Messages -| Type | Struct Name | Description | -| :--: | --------------------------------- | ---------------------------------------------------------------------- | -| `A` | `AddOrderMessage` | A new order has been accepted (no MPID attribution). | -| `F` | `AddOrderMPIDMessage` | A new order has been accepted (with MPID attribution). | -| `E` | `OrderExecutedMessage` | An order on the book was executed in part or in full. | -| `C` | `OrderExecutedWithPriceMessage` | An order was executed at a price different from the display price. | -| `X` | `OrderCancelMessage` | An order was partially canceled; this message contains the canceled quantity. | -| `D` | `OrderDeleteMessage` | An order was canceled in its entirety and removed from the book. | -| `U` | `OrderReplaceMessage` | An existing order has been replaced with a new order. | - +| Type | Struct Name | Description | +| :--: | ------------------------------- | ----------------------------------------------------------------------------- | +| `A` | `AddOrderMessage` | A new order has been accepted (no MPID attribution). | +| `F` | `AddOrderMPIDMessage` | A new order has been accepted (with MPID attribution). | +| `E` | `OrderExecutedMessage` | An order on the book was executed in part or in full. | +| `C` | `OrderExecutedWithPriceMessage` | An order was executed at a price different from the display price. | +| `X` | `OrderCancelMessage` | An order was partially canceled; this message contains the canceled quantity. | +| `D` | `OrderDeleteMessage` | An order was canceled in its entirety and removed from the book. | +| `U` | `OrderReplaceMessage` | An existing order has been replaced with a new order. | ### Trade Messages -| Type | Struct Name | Description | -| :--: | -------------------- | ----------------------------------------------------------------------------- | -| `P` | `TradeMessage` | Reports a trade for a non-displayable order type. | -| `Q` | `CrossTradeMessage` | Reports a cross trade (e.g., Opening, Closing, IPO Cross). | -| `B` | `BrokenTradeMessage` | Reports that a previously disseminated execution has been broken. | - +| Type | Struct Name | Description | +| :--: | -------------------- | ----------------------------------------------------------------- | +| `P` | `TradeMessage` | Reports a trade for a non-displayable order type. | +| `Q` | `CrossTradeMessage` | Reports a cross trade (e.g., Opening, Closing, IPO Cross). | +| `B` | `BrokenTradeMessage` | Reports that a previously disseminated execution has been broken. | ### Imbalance and Price Discovery Messages -| Type | Struct Name | Description | -| :--: | -------------------------------------------- | ---------------------------------------------------------------------------- | -| `K` | `IPOQuotingPeriodUpdateMessage` | Provides the anticipated IPO quotation release time. | -| `J` | `LULDAuctionCollarMessage` | Indicates auction collar thresholds for a security in a LULD Trading Pause. | -| `I` | `NOIIMessage` | Net Order Imbalance Indicator message, used for opening/closing crosses. | -| `N` | `RetailPriceImprovementIndicator` | Indicates the presence of Retail Price Improvement (RPII) interest. | -| `O` | `DirectListingCapitalRaisePriceDiscovery` | Disseminated for Direct Listing with Capital Raise (DLCR) securities. | +| Type | Struct Name | Description | +| :--: | ----------------------------------------- | --------------------------------------------------------------------------- | +| `K` | `IPOQuotingPeriodUpdateMessage` | Provides the anticipated IPO quotation release time. | +| `J` | `LULDAuctionCollarMessage` | Indicates auction collar thresholds for a security in a LULD Trading Pause. | +| `I` | `NOIIMessage` | Net Order Imbalance Indicator message, used for opening/closing crosses. | +| `N` | `RetailPriceImprovementIndicator` | Indicates the presence of Retail Price Improvement (RPII) interest. | +| `O` | `DirectListingCapitalRaisePriceDiscovery` | Disseminated for Direct Listing with Capital Raise (DLCR) securities. | --- @@ -496,16 +502,37 @@ This library supports all message types defined in the ITCH 5.0 specification. T The parser is designed for high-throughput scenarios. Performance is heavily dependent on the underlying hardware (CPU speed, memory bandwidth, and I/O speed). -To run the performance benchmarks: +### Benchmarks + +The numbers below were produced by [`benchmarks/parser_bench.cpp`](benchmarks/parser_bench.cpp) parsing from an in-memory buffer (file I/O excluded). They are reproducible with the setup described under [How to reproduce](#how-to-reproduce). + +| Configuration | Throughput | +| ------------- | ---------- | +| `parse(..., callback)` — allocation-free callback path | **2.24 GiB/s** | +| `parse(...)` — collect all messages into a `std::vector` | 1.23 GiB/s | +| `parse(..., {'A','P','E','C','X'})` — filtered into a `std::vector` | 1.15 GiB/s | + +- **Hardware**: x86-64, 16 logical cores @ 1.70 GHz (L1d 32 KiB, L2 512 KiB, L3 4 MiB). +- **Compiler**: Clang 22, `-DCMAKE_BUILD_TYPE=Release`, C++23. +- **Dataset**: official NASDAQ TotalView-ITCH 5.0 sample (`01302020`), a ~300 MB frame-aligned slice loaded fully into memory. + +The collect/filter paths are slower than the callback path because they allocate and copy each parsed message into a `std::vector`; the callback path does neither, which is why it is the recommended interface for latency-sensitive processing. + +### How to reproduce + ```bash -# Configure and build with benchmarks enabled -cmake -S . -B build -DITCH_BUILD_BENCHMARKS=ON +# 1. Fetch an official sample (multi-GB; not committed to the repo). +scripts/fetch_sample.sh data + +# 2. Configure and build with benchmarks enabled. +cmake -S . -B build -DITCH_BUILD_BENCHMARKS=ON -DITCH_CXX_STANDARD=23 -DCMAKE_BUILD_TYPE=Release cmake --build build --config Release -# Run the benchmark suite -./build/benchmarks/parser_bench +# 3. Run the benchmark suite against the sample. +./build/benchmarks/parser_bench data/01302020.NASDAQ_ITCH50 ``` -The output will show the parsing rate in messages per second and throughput in GB/s. On modern server hardware, expect throughput in the multi-gigabytes per second range when parsing from an in-memory buffer. + +The output reports throughput in GiB/s for each parsing strategy. --- @@ -516,14 +543,15 @@ The project uses `.clang-format` to enforce a consistent code style. A `.clang-t --- ## Frequently Asked Questions (FAQ) + - **What exactly is this NASDAQ ITCH data?** -It's the most detailed data feed available from NASDAQ. It's not just stock prices; it’s every single order placed, modified, cancelled, and executed. It's the complete "play-by-play" of the market. + It's the most detailed data feed available from NASDAQ. It's not just stock prices; it’s every single order placed, modified, cancelled, and executed. It's the complete "play-by-play" of the market. - **Why can't I just use a simple program to read this data?** -The data is in a highly optimized, machine-only binary format, not human-readable text. More importantly, the volume and velocity are immense—a single day of trading can generate tens or hundreds of gigabytes of data. A standard program would be far too slow to keep up. + The data is in a highly optimized, machine-only binary format, not human-readable text. More importantly, the volume and velocity are immense—a single day of trading can generate tens or hundreds of gigabytes of data. A standard program would be far too slow to keep up. - **Is this a program I can just double-click and run?** -No, it’s a specialized component—a library—that software developers use as the "engine" inside a larger application (like a trading platform or an analysis tool). It does the heavy lifting of data processing so they can focus on building the features their users need. + No, it’s a specialized component—a library—that software developers use as the "engine" inside a larger application (like a trading platform or an analysis tool). It does the heavy lifting of data processing so they can focus on building the features their users need. --- @@ -537,4 +565,4 @@ This project is licensed under the MIT License. See the [LICENSE](LICENSE) file ## References -* **Nasdaq TotalView-ITCH 5.0 Specification:** The official [documentation](https://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NQTVITCHspecification.pdf) is the definitive source for protocol details. +- **Nasdaq TotalView-ITCH 5.0 Specification:** The official [documentation](https://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NQTVITCHspecification.pdf) is the definitive source for protocol details. From 181f1e044e7f570598c8b93fc27ea05c2375e7ef Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sun, 28 Jun 2026 20:14:12 +0100 Subject: [PATCH 026/144] Add Phase 1 real feed ingestion (transport decoders) Implement the FEATURES.md Phase 1 transport layer so the library can consume ITCH as it is actually delivered, not only pre-stripped message streams: - MoldUdp64Decoder: UDP multicast framing (session, sequence, message blocks). - SoupBinDecoder: TCP framing for Glimpse/recovery, buffering partial segments. - PcapReader: in-house .pcap/.pcapng reader (no libpcap) walking Ethernet/IPv4/IPv6/UDP into the MoldUDP64 decoder, with a UDP port filter. - SequenceTracker + RetransmitRequester: per-session gap detection and a caller-driven recovery hook. Adds round-trip transport tests, a transport fuzz target, a pcap replay example, README/CHANGELOG entries, and wires the new fuzzer into CI. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build.yml | 9 +- CHANGELOG.md | 89 ++++++ CLAUDE.md | 89 ++++++ FEATURES.md | 153 ++++++++++ IMPROVEMENTS.md | 330 +++++++++++++++++++++ README.md | 39 +++ VERSION.txt | 3 +- examples/CMakeLists.txt | 8 + examples/transport/pcap_replay_example.cpp | 50 ++++ fuzz/CMakeLists.txt | 7 + fuzz/moldudp64_fuzzer.cpp | 28 ++ include/itch/transport/moldudp64.hpp | 95 ++++++ include/itch/transport/pcap.hpp | 82 +++++ include/itch/transport/sequencing.hpp | 130 ++++++++ include/itch/transport/soupbintcp.hpp | 93 ++++++ src/CMakeLists.txt | 9 +- src/transport/moldudp64.cpp | 72 +++++ src/transport/pcap.cpp | 255 ++++++++++++++++ src/transport/soupbintcp.cpp | 157 ++++++++++ tests/CMakeLists.txt | 3 + tests/transport/frame_builders.hpp | 163 ++++++++++ tests/transport/test_transport.cpp | 183 ++++++++++++ 22 files changed, 2039 insertions(+), 8 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 CLAUDE.md create mode 100644 FEATURES.md create mode 100644 IMPROVEMENTS.md create mode 100644 examples/transport/pcap_replay_example.cpp create mode 100644 fuzz/moldudp64_fuzzer.cpp create mode 100644 include/itch/transport/moldudp64.hpp create mode 100644 include/itch/transport/pcap.hpp create mode 100644 include/itch/transport/sequencing.hpp create mode 100644 include/itch/transport/soupbintcp.hpp create mode 100644 src/transport/moldudp64.cpp create mode 100644 src/transport/pcap.cpp create mode 100644 src/transport/soupbintcp.cpp create mode 100644 tests/transport/frame_builders.hpp create mode 100644 tests/transport/test_transport.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6cd3d9e..93e64e0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -253,12 +253,15 @@ jobs: -DCMAKE_C_COMPILER=clang \ -DCMAKE_CXX_COMPILER=clang++ - - name: Build fuzzer - run: cmake --build $BUILD_DIR --target parser_fuzzer + - name: Build fuzzers + run: cmake --build $BUILD_DIR --target parser_fuzzer moldudp64_fuzzer - - name: Run fuzzer (time-budgeted) + - name: Run parser fuzzer (time-budgeted) run: $BUILD_DIR/fuzz/parser_fuzzer -max_total_time=60 -max_len=512 -print_final_stats=1 + - name: Run transport fuzzer (time-budgeted) + run: $BUILD_DIR/fuzz/moldudp64_fuzzer -max_total_time=60 -max_len=2048 -print_final_stats=1 + benchmark-smoke: runs-on: ubuntu-latest env: diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..c0a44be --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,89 @@ +# Changelog + +All notable changes to this project are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +- **Phase 1 — Real feed ingestion** ([FEATURES.md](FEATURES.md)). The library can + now consume ITCH as it is actually delivered, not only pre-stripped message + streams: + - `itch::transport::MoldUdp64Decoder` (`itch/transport/moldudp64.hpp`) decodes + the UDP multicast framing NASDAQ uses for live dissemination: it unpacks the + session, sequence number, and message blocks of each datagram and feeds them + to the existing parser, handling heartbeat and end-of-session packets. + - `itch::transport::SoupBinDecoder` (`itch/transport/soupbintcp.hpp`) decodes + the TCP framing used for the Glimpse snapshot and recovery/replay services, + including login, heartbeats, logout, end-of-session, and sequenced/unsequenced + data, buffering partial packets across stream segments. + - `itch::transport::PcapReader` (`itch/transport/pcap.hpp`) replays a feed + straight from a captured `.pcap`/`.pcapng` file. It is implemented in-house + with no libpcap dependency, walking the Ethernet/IPv4/IPv6/UDP layers and + feeding the payload through the MoldUDP64 decoder, with an optional UDP + destination-port filter. + - `itch::transport::SequenceTracker` and `RetransmitRequester` + (`itch/transport/sequencing.hpp`) track per-session sequence numbers across + the transport layer, surface gaps through a callback, and expose a re-request + hook a caller can implement to drive recovery against a replay server. + - A transport fuzz target (`fuzz/moldudp64_fuzzer.cpp`) and a `pcap` replay + example (`examples/transport/pcap_replay_example.cpp`). + +- `std::span` overloads of `Parser::parse` as the preferred, + size-carrying buffer interface. The `(const char*, size_t)` overloads remain as + thin shims for C interop. (IMPROVEMENTS 3.1) +- Non-throwing `Parser::try_parse` entry points returning + `std::expected<…, ParseError>` for latency-sensitive callers that prefer not to + use exceptions on the hot path. Available when the standard library provides + `std::expected` (C++23). (IMPROVEMENTS 3.2) +- An explicit error policy on `Parser`: `set_error_callback`, the + `unknown_message_count` / `malformed_message_count` diagnostics counters, and + `reset_diagnostics`. (IMPROVEMENTS 1.2) +- Per-message length validation: a frame whose declared length is smaller than its + message type requires is now rejected (counted, and reported to the error + callback) instead of being decoded into the following frame. (IMPROVEMENTS 1.1) +- Strongly typed fixed-point `Price` (`itch::StandardPrice`, `itch::MwcbPrice`) in + `itch/price.hpp`, encoding the decimal scale in the type so the 4-vs-8 decimal + distinction cannot be mixed up. (IMPROVEMENTS 3.3) +- Timestamp-to-wall-clock helpers in `itch/time.hpp` (`to_time_point`, + `format_timestamp`, `format_time_point`). (IMPROVEMENTS 3.4) +- Parser fuzz target (`fuzz/parser_fuzzer.cpp`, `ITCH_BUILD_FUZZERS`) and a + time-budgeted CI fuzzing job. (IMPROVEMENTS 5.4) +- Published, reproducible benchmark numbers in the README and a CI benchmark smoke + job. (IMPROVEMENTS 2.4) +- `scripts/fetch_sample.sh` to download an official ITCH sample on demand. + +### Changed + +- **Behavior change:** the library no longer writes to `std::cerr` on an unknown + message type. Unknown types are now silently skipped, counted, and surfaced + through the optional error callback. Callers that relied on the stderr output + should install an error callback or read the diagnostics counters. + (IMPROVEMENTS 1.2) +- Message dispatch now uses a compile-time, type-byte-indexed flat table instead of + a `std::map>`, removing the per-message tree lookup and + type-erased call. (IMPROVEMENTS 2.1) +- Byte swapping now uses `std::byteswap` / `std::bit_cast` instead of union + type-punning, eliminating undefined behavior on the hot path. (IMPROVEMENTS 1.3) +- Order-book rendering and indicator lookups now use `std::format` and `constexpr` + frozen tables instead of iostream manipulators and namespace-scope `std::map` + globals. (IMPROVEMENTS 4.2, 4.3) +- Standardized the minimum CMake version on 3.25 across the project. + (IMPROVEMENTS 5.2) +- README architecture section corrected to describe the parser as a fast, + allocation-free, single-pass field decoder rather than a "zero-copy / + zero-overhead" deserializer. (IMPROVEMENTS 2.2) + +### Fixed + +- Renamed misspelled public CMake helpers: `anable_address_sanitizer` → + `enable_address_sanitizer`, `build_documenation` → `build_documentation`, + `apply_formating` → `apply_formatting`, `apply_clang_tidy_globaly` → + `apply_clang_tidy_globally`. (IMPROVEMENTS 5.5) +- CI now runs on pull requests and adds clang-format, clang-tidy, ASAN/UBSAN, and + coverage jobs. (IMPROVEMENTS 5.3) +- Stopped tracking the generated `docs/coverage.html`; coverage is now generated + into the build tree only. (IMPROVEMENTS 5.1) diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ca8a03f --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,89 @@ +# Modern C++ Best Practices + +## Language & Compiler + +- Target C++17 minimum; prefer C++20/23. Set `CMAKE_CXX_STANDARD` explicitly; disable extensions (`-pedantic`). +- Enable `-Wall -Wextra -Wconversion -Wsign-conversion` and treat warnings as errors (`-Werror`). + +## Style & Naming + +- `m_` prefix for private members; `snake_case` for variables/functions; `PascalCase` for types; `SCREAMING_SNAKE_CASE` for macros and Constants. +- All symbols (variable, parameters, functions) must be at least 3 characters long. +- Alwayse Use `std::print`/`std::println`/`std::format` for output. +- No magic literals, replace with named `constexpr` values. Declare variables close to first use. +- Keep functions short and flat; remove unused variables, parameters, and includes. +- Use `#pragma once`; keep headers to declarations and inline/template definitions only. +- Do not use mdash (`--`) in code, comments, or documentation. + +## Variables & Types + +- Always initialize variables at declaration; prefer brace initialization `T x{value}` to prevent narrowing. +- Use `auto` when type is obvious; `const` by default; `constexpr` for compile-time constants. +- Avoid `using namespace std` in headers. Use `enum class`; use `std::string_view` for read-only strings. +- Never mix signed/unsigned arithmetic; never use C-style casts, use `static_cast`, `dynamic_cast`, etc. +- Never use raw pointers for ownership; use references or smart pointers instead. Avoid `void*` and `reinterpret_cast`. +- Never use `goto` or `longjmp`; prefer structured control flow and RAII for cleanup. +- Never use C-style arrays or C-style strings; prefer `std::array`, `std::vector`, and `std::string` for safety and convenience. + +## Memory Management + +- Follow RAII: tie every resource (memory, file, mutex) to an object's lifetime. +- Use `std::unique_ptr`/`std::shared_ptr`/`std::weak_ptr` via `make_unique`/`make_shared`, never `new`/`delete` directly. +- Follow Rule of Zero (all RAII members) or Rule of Five (when owning a raw resource). +- Detect leaks with AddressSanitizer; use standard containers over raw arrays. + +## Functions & Classes + +- One logical operation per function; always use trailing return types. +- Pass non-trivial types as `const T&`; return by value (rely on NRVO/RVO). +- Mark `[[nodiscard]]` where return values must not be ignored; mark `noexcept` when applicable. +- Single-argument constructors must be `explicit`; keep members `private`; use `override`/`final` on overrides. +- Prefer composition over inheritance; declare base destructors `virtual` (or `protected` non-virtual). +- Do not call virtual functions from constructors or destructors. + +## Error Handling + +- Use exceptions for errors (not for normal control flow); catch by `const` reference to avoid slicing. +- Maintain a consistent exception-safety guarantee: basic, strong, or no-throw. +- Use `static_assert` for compile-time invariants; `assert()` for debug-only preconditions, never on user input. +- Consider `std::expected` (C++23) for predictable, performance-sensitive failure paths. + +## Templates & Concurrency + +- Constrain templates with C++20 concepts; use `if constexpr` over SFINAE/`enable_if`. +- Use variadic templates and fold expressions instead of recursive specializations. +- Protect shared mutable state with `std::mutex`; prefer `std::jthread` over `std::thread`. +- Use `std::atomic` for lock-free single-variable operations; never `volatile` for inter-thread communication. +- Minimize lock duration; avoid callbacks or virtual calls while holding a lock; prefer `std::async` and parallel algorithms. + +## Performance & Build + +- Profile before optimizing; prefer `std::vector` for cache-friendly access; use `reserve()` when size is known. +- Use move semantics to avoid deep copies; prefer standard algorithms (`std::sort`, `std::transform`) over hand-written loops. +- Use CMake (≥ 3.25) with target-centric paradigm: `target_include_directories`, `target_compile_options`, `target_link_libraries`. +- Use `FetchContent` or vcpkg/Conan for dependencies; use out-of-source builds and CMake Presets. +- Enable `CMAKE_EXPORT_COMPILE_COMMANDS=ON` for tooling; use `ccache`/`sccache` to speed up rebuilds. + +## Testing & Tooling + +- Use GoogleTest/Catch2 with CTest; write tests before or alongside implementation. +- Run tests under AddressSanitizer and UndefinedBehaviourSanitizer; use fuzz testing for parsers and untrusted input. +- Enforce clang-format in CI; use clang-tidy with `cppcoreguidelines-*`, `modernize-*`, `readability-*` checks. +- Use debuggers (GDB, LLDB) actively rather than `std::cout`; use `compile_commands.json` for accurate tooling. +- Use real implementations; only mock external dependencies (LLM APIs, cloud services). +- Test public interface behavior, not implementation details, keeps tests resilient to refactoring. +- Tests must be fast, isolated, and have descriptive names; cover edge cases and error conditions. +- Location: `tests/unittests/` following source structure. + +## CI/CD & Maintenance + +## Versioning & Breaking Changes + +- Follow Semantic Versioning 2.0.0 (`MAJOR.MINOR.PATCH`). +- A breaking change is any backward-incompatible modification to the public API, including data schemas, CLI, and server communication formats. +- CI must build and test before merging; build across GCC, Clang, and MSVC. +- Document the _why_, not the _what_; use Doxygen (`///`) for API docs and `//` for implementation notes. +- Keep public API surface minimal; mark deprecated APIs with `[[deprecated("reason")]]` and include a migration path. +- Follow the Boy Scout Rule; track technical debt explicitly; maintain a `CHANGELOG.md` and `CONTRIBUTING.md`. + +--- diff --git a/FEATURES.md b/FEATURES.md new file mode 100644 index 0000000..b1b3e22 --- /dev/null +++ b/FEATURES.md @@ -0,0 +1,153 @@ +# FEATURES + +This document is the product roadmap for ITCHCPP. Where [IMPROVEMENTS.md](IMPROVEMENTS.md) +is about making the existing parser correct, fast, and honest, this file is about +the new capabilities that turn a NASDAQ ITCH 5.0 file parser into a market-data +platform that quantitative teams, trading firms, and researchers can rely on +everywhere in their workflow. + +## Vision + +> A single, modern C++ library that ingests real exchange market data — live or +> historical — reconstructs the book, exposes microstructure analytics, and feeds +> both production trading systems and Python research notebooks, with the speed +> expected of latency-sensitive infrastructure. + +Today ITCHCPP parses the raw, length-prefixed ITCH 5.0 message stream and +reconstructs a single-symbol order book. To reach the vision it needs to read the +data formats that actually arrive on the wire and on disk, scale the book to whole +markets, compute the metrics quants ask for, and meet researchers in the tools +they already use. + +The roadmap is phased. Each phase is independently valuable and shippable; later +phases build on earlier ones. The order reflects what unlocks the most real-world +usage first. + +--- + +## Phase 1 — Real feed ingestion + +_Goal: make ITCHCPP usable on data as it actually exists, not just pre-stripped +message streams._ + +The current parser assumes a raw concatenation of length-prefixed messages. Real +ITCH is delivered inside transport framing. Without these decoders the library +cannot consume a live feed or most captured data. + +- **MoldUDP64 decoder** — the UDP multicast framing NASDAQ uses for live + dissemination. Unpacks sessions, sequence numbers, and message blocks, then + feeds the existing message parser. This is the gateway to live data. +- **SoupBinTCP decoder** — the TCP framing used for the glimpse snapshot and + recovery/replay services. Handles login, heartbeats, and sequenced payloads. +- **pcap ingestion** — replay market data straight from captured network traffic + (`.pcap`/`.pcapng`), the most common form in which firms archive and share feeds. +- **Gap detection and recovery hooks** — track sequence numbers across the + transport layer, surface gaps, and expose re-request hooks so a caller can drive + recovery against a replay server. + +--- + +## Phase 2 — Production order-book engine + +_Goal: reconstruct the full market, fast, not just one symbol._ + +The current `LimitOrderBook` is constructed for a single symbol +([include/itch/order_book.hpp:75](include/itch/order_book.hpp)) and is allocation- +heavy (see [IMPROVEMENTS.md](IMPROVEMENTS.md) 2.3). A serious book engine handles +every symbol on the feed at line rate. + +- **Multi-symbol book manager** — maintain books for all (or a selected universe + of) symbols from one pass over the feed, keyed by stock locate code for O(1) + routing. +- **Allocation-free internals** — order object pool / arena, intrusive FIFO queues + per price level, and flat price ladders, with O(1) lookup of orders by reference + number. Removes the per-order heap and refcount cost. (Deferred book rewrite + tracked from [IMPROVEMENTS.md](IMPROVEMENTS.md) 2.3; the current `std::shared_ptr` + book is correct but allocation-heavy.) +- **Zero-copy overlay parse API** — an alternative to the current single-pass field + decoder: typed views over the raw buffer with lazy, on-access endianness + conversion, for callers that touch only a few fields per message. The library + today decodes eagerly into host-order structs; this overlay is tracked from + [IMPROVEMENTS.md](IMPROVEMENTS.md) 2.2 as the stronger product direction. +- **BBO and top-of-book change events** — emit best-bid/offer updates as they + happen, the primary signal most consumers need. +- **Depth snapshots** — configurable L2 aggregated depth and full L3 order-level + detail, on demand or on every change. +- **Trade tape extraction** — a clean stream of executions, correctly + distinguishing displayable from non-displayable/hidden prints using the + printable flag. + +--- + +## Phase 3 — Analytics & microstructure + +_Goal: answer the questions quants actually ask, in the library, correctly._ + +Once the book and tape are reconstructed, the high-value layer is derived metrics. +Computing them centrally and correctly saves every downstream team from +reimplementing them. + +- **Bar builders** — OHLCV aggregation over time, volume, and tick clocks. +- **VWAP / TWAP** — running and interval volume- and time-weighted average prices. +- **Order-flow imbalance and microstructure metrics** — spread, depth at level, + queue imbalance, and order-flow imbalance, the staples of short-horizon research. +- **NOII / imbalance analytics** — surface the net order imbalance indicator data + the feed already carries in a usable form. +- **Auction reconstruction** — opening, closing, halt, and IPO cross price + discovery from the cross and NOII messages. + +--- + +## Phase 4 — Research & interoperability + +_Goal: meet quants where they live. This is what "everywhere in finance" means._ + +Most quantitative research happens in Python and columnar data tools. A +C++-only library, however fast, is invisible to that workflow until it has +bindings and standard export formats. + +- **Python bindings (pybind11)** — first-class, zero-friction access from Python, + exposing parsing, the book engine, and analytics. The single highest-leverage + feature for adoption. + For this , we already have a pure python library [itch](https://github.com/bbalouki/itch) so we want this to become a pure binding of the current library. But since it already has been published on pypi , we don't know the best action to take you decide. + Next is the itch message documentation, the current c++ version doe not docoment each message, use the pytohn version to read each message documenation and write it on the C++ using doxygen style (///) and (///<) +- **Apache Arrow / Parquet export** — turn a feed into columnar tables that drop + straight into pandas, Polars, DuckDB, and Spark pipelines. +- **CSV and streaming sinks** — simple, universal output for quick inspection and + legacy tooling. +- **`itch-tool` CLI** — a batteries-included command-line tool to inspect, filter, + convert (e.g. ITCH to Parquet), and produce message-type statistics/histograms + without writing any code. + +--- + +## Phase 5 — Simulation & ecosystem + +_Goal: close the loop for backtesting and become a maintained, depended-on package._ + +- **Replay engine with timestamp pacing** — drive consumers at original or scaled + wall-clock speed for realistic backtesting and system simulation. +- **ITCH encoder / writer** — synthesize valid ITCH streams for testing, scenario + generation, and golden fixtures (also unblocks much of the testing work in + [IMPROVEMENTS.md](IMPROVEMENTS.md) 6). +- **Multi-venue and multi-version support** — generalize via policy templates to + NASDAQ BX/PSX, older ITCH 4.1, and other ITCH-like venue feeds, so one codebase + covers a firm's whole equity data estate. +- **Packaging and project maturity** — publish to vcpkg and Conan, define a + versioned ABI/compatibility policy, and maintain a `CHANGELOG.md` and + `CONTRIBUTING.md` so the project is safe to depend on. + +--- + +## Non-goals + +To stay focused and best-in-class at what it does, ITCHCPP deliberately will not +become: + +- An order-management or execution-management system (OMS/EMS). +- An exchange matching engine. +- A strategy/alpha research framework or backtester proper (it _feeds_ those; it + is not one). + +Staying a superb data layer — ingest, reconstruct, analyze, export — is the way to +be used everywhere rather than competing with the systems built on top of it. diff --git a/IMPROVEMENTS.md b/IMPROVEMENTS.md new file mode 100644 index 0000000..b8db94c --- /dev/null +++ b/IMPROVEMENTS.md @@ -0,0 +1,330 @@ +# IMPROVEMENTS + +This document is a prioritized audit of the current ITCHCPP codebase. It exists +to chart the path from a capable single-protocol file parser to infrastructure +that can credibly claim to be "one of the best in quantitative finance" and be +"usable everywhere in finance." + +Every finding below is tied to a specific location in the current source so it +can be acted on directly. Items are tagged by priority: + +- **P0** — correctness, safety, or a claim/reality gap that undermines trust. Fix first. +- **P1** — meaningful quality, performance, or process gaps. Fix soon. +- **P2** — polish and consistency. Fix opportunistically (Boy Scout Rule). + +New capabilities (transport, analytics, bindings, etc.) live in +[FEATURES.md](FEATURES.md). This file is about making what already exists correct, +fast, honest, and maintainable. + +--- + +## 1. Correctness & specification compliance + +### 1.1 Per-message length is never validated against the expected message size — P0 + +**Problem.** The framing loop reads the 2-byte length prefix, checks only that the +declared payload fits in the buffer, then dispatches on the type byte +([src/parser.cpp:291-322](src/parser.cpp)). The per-type `unpack_*` functions read +a fixed number of bytes based on the struct layout, never cross-checked against the +declared `length`. A message whose `length` is shorter than its type implies (a +corrupt or maliciously crafted frame) still dispatches, and the unpacker reads +fields belonging to the _next_ frame. + +**Impact.** Silent misparsing of adjacent data with no error raised. In a finance +context this is the worst failure mode: wrong numbers that look plausible. It is +also a robustness hole for untrusted input. + +**Fix.** Maintain a compile-time table of expected payload size per message type +and validate `length` against it before unpacking. On mismatch, route to the error +policy (see 1.2) rather than proceeding. This pairs naturally with the dispatch +rewrite in 2.1. + +### 1.2 The library prints to `std::cerr` on unknown message types — P0 + +**Problem.** When an unknown type byte is encountered the parser writes to +`std::cerr` ([src/parser.cpp:318](src/parser.cpp)). + +**Impact.** A library must never write to a global stream on a caller's behalf: it +corrupts the host application's output, is not thread-safe, and cannot be silenced. +It also violates the project's own house rule favoring `std::print`/`std::format` +over stream I/O. + +**Fix.** Replace with an explicit error policy: an optional error callback, an +accumulating diagnostics counter, or a `std::expected` return on the typed-parse +path. Default behavior should be silent-skip-and-count, configurable by the caller. + +### 1.3 `swap_bytes` relies on union type-punning — P0 + +**Problem.** Byte swapping is implemented by writing one union member and reading +another ([include/itch/parser.hpp:150-161](include/itch/parser.hpp)). Reading an +inactive union member is undefined behavior in C++ (it is only well-defined in C). + +**Impact.** Works today by compiler grace, but it is UB on the parser hot path — +exactly the code that must be bulletproof. Aggressive optimizers are within their +rights to miscompile it. + +**Fix.** Use `std::byteswap` (C++23, already a target standard) with a `std::endian` +check; fall back to `std::bit_cast` or compiler intrinsics +(`__builtin_bswap*` / `_byteswap_*`) for C++20. + +--- + +## 2. Performance — the headline gap versus the README + +The README promises "multi-gigabyte-per-second" throughput and "zero-overhead +deserialization." The implementation does not currently match that framing. These +items close the gap, or correct the claims. + +### 2.1 Dispatch uses `std::map>` — P0 + +**Problem.** Message dispatch is a `std::map>` +([include/itch/parser.hpp:133-134](include/itch/parser.hpp), +[src/parser.cpp:250-322](src/parser.cpp)). Every single message pays for a +red-black-tree lookup (pointer chasing, O(log n)) _plus_ a `std::function` +type-erased call (indirect call, possible heap, no inlining). + +**Impact.** This is the single biggest divergence from the "gigabytes/second, +zero-overhead" promise. On a feed of hundreds of millions of messages per day this +dominates the cost. + +**Fix.** Dispatch on the type byte through a 256-entry flat function-pointer table +or a `switch`. Type bytes are dense ASCII, so a flat array is cache-friendly and +branch-predictable, and lets the compiler inline each unpacker. Combine with 1.1's +size table so dispatch and validation share one lookup. + +### 2.2 The "zero-copy / zero-overhead deserialization" claim is inaccurate — P0 + +**Problem.** The README's architecture section states the parser reads messages +"without performing heavy data conversions" and copies bytes "skipping the usual +per-field decoding steps" ([README.md:83-85](README.md)). The actual code does +field-by-field `memcpy` plus per-field byte swap into separate host-order structs +([src/parser.cpp:69-263](src/parser.cpp)). That is per-field decoding, not +zero-copy. + +**Impact.** Overstated claims are a credibility risk with an infrastructure +audience that will read the source. + +**Fix.** Either (a) implement a genuine zero-copy overlay API that returns typed +views over the raw buffer with lazy, on-access endianness conversion, or +(b) correct the wording to describe what it does: a fast, allocation-free, +single-pass field decoder. Option (a) is the stronger product; see +[FEATURES.md](FEATURES.md). + +### 2.3 The order book is allocation-heavy — P1 + +**Problem.** Each resting order is a `std::shared_ptr` stored in a +`std::list`, with price levels held in `std::map` +([include/itch/order_book.hpp:25-61](include/itch/order_book.hpp), +[src/order_book.cpp](src/order_book.cpp)). That is an atomic refcount and a heap +allocation per order, plus a node allocation per list entry and per map node. + +**Impact.** This directly contradicts the zero-allocation positioning and makes the +book the bottleneck for any real reconstruction workload (book rebuilds touch every +add/cancel/execute/replace message). + +**Fix.** Move to an object pool / arena for orders, intrusive FIFO lists at each +price level, and flat (vector-backed) price ladders with O(1) order lookup by +reference number. Tracked in detail under the Phase 2 book engine in +[FEATURES.md](FEATURES.md). + +### 2.4 No published, reproducible benchmark numbers — P1 + +**Problem.** A Google Benchmark suite exists +([benchmarks/parser_bench.cpp](benchmarks/parser_bench.cpp)) but the README quotes +no concrete results, only aspirational ranges. + +**Impact.** "High performance" is unverifiable, and there is no regression signal +when 2.1–2.3 land. + +**Fix.** Publish a results table (hardware, compiler, dataset, msgs/sec, GB/s) and +wire the benchmark into CI to catch regressions. + +--- + +## 3. API modernization + +### 3.1 Replace `(const char*, size_t)` with `std::span` — P1 + +**Problem.** The buffer APIs take raw pointer/size pairs +([include/itch/parser.hpp:61-91](include/itch/parser.hpp)). + +**Fix.** Accept `std::span`. It carries the size, prevents +pointer/length desync, and is the modern idiom the project's standards call for. +Keep the pointer/size overloads as thin shims for C interop. + +### 3.2 Offer a non-throwing `std::expected` parse path — P1 + +**Problem.** Errors are reported only via exceptions (`std::runtime_error` on +truncation, [src/parser.cpp:296-309](src/parser.cpp)). + +**Impact.** Exceptions on the hot path are awkward for latency-sensitive callers, +and the project's standards explicitly suggest `std::expected` for predictable +failure paths. + +**Fix.** Provide a `std::expected`-based entry point alongside the +throwing wrappers. + +### 3.3 Introduce a typed fixed-point `Price` — P1 + +**Problem.** Prices are exposed as raw `uint32_t` and the caller is expected to +divide by `PRICE_DIVISOR` (10000) themselves +([include/itch/messages.hpp:327-329](include/itch/messages.hpp)), with the further +trap that MWCB decline levels use 8 decimals, not 4. + +**Impact.** Easy to misuse; mixing the two scales silently produces wrong prices. + +**Fix.** Add a strong `Price` type encoding scale, with explicit conversions to +`double`/decimal. Make the 4-vs-8 decimal distinction unrepresentable as a bug. + +### 3.4 Provide timestamp-to-wall-clock helpers — P1 + +**Problem.** Timestamps are exposed as raw nanoseconds-past-midnight `uint64_t` +with no utilities to combine them with a session date. + +**Fix.** Add helpers that convert to `std::chrono` time points given a session +date, plus formatting helpers. + +--- + +## 4. Adherence to the project's own C++ standards + +The repository ships a detailed [CLAUDE.md](CLAUDE.md) of C++ standards. Several +are violated by the current code. + +### 4.1 Symbols shorter than three characters — P2 + +**Problem.** The standard requires every symbol be at least three characters, yet +the code uses `val`, `src`, `dst`, `arr`, `sv`, single-letter loop indices `i`, +and template parameter `T` (e.g. +[include/itch/parser.hpp:150-161](include/itch/parser.hpp), +[include/itch/indicators.hpp:9](include/itch/indicators.hpp)). + +**Fix.** Rename to descriptive identifiers during the relevant refactors. + +### 4.2 Stream I/O instead of `std::print`/`std::format` — P2 + +**Problem.** The standard mandates `std::print`/`std::println`/`std::format`, but +output uses `std::cout`/`std::cerr` and iostream manipulators +([src/order_book.cpp:29-68](src/order_book.cpp), [src/parser.cpp:318](src/parser.cpp)). + +**Fix.** Migrate output paths to `std::format`/`std::print` (overlaps with 1.2). + +### 4.3 `std::map` globals defined in a header — P2 + +**Problem.** [include/itch/indicators.hpp](include/itch/indicators.hpp) defines +several non-trivially-constructed `std::map` objects at namespace scope in a header. +Each translation unit that includes it pays dynamic-initialization cost, and the +maps risk duplication. + +**Fix.** Replace with `constexpr` frozen lookups (sorted `std::array` of pairs with +binary search, or a compile-time map type) or functions returning `string_view`, +keeping the header dependency-light. + +--- + +## 5. Build, CI, and repository hygiene + +### 5.1 Large binary artifacts committed to the repository — P1 + +**Problem.** The repo tracks a full uncompressed NASDAQ ITCH sample +(`01302020.NASDAQ_ITCH50`), its `.gz` counterpart, and a generated coverage report +(`docs/coverage.html`). + +**Impact.** Bloated clone size, slow history, and a generated artifact under source +control. The data files in particular are large binaries that do not belong in git. + +**Fix.** Remove from history (or migrate to Git LFS / an external download script), +add them to `.gitignore`, and generate coverage into the build tree only. + +### 5.2 Inconsistent minimum CMake version — P2 + +**Problem.** The required CMake version disagrees across the project: README says +3.20, [cmake/Helpers.cmake:1](cmake/Helpers.cmake) says 3.23, and +[CLAUDE.md](CLAUDE.md) asks for >= 3.25. + +**Fix.** Standardize on a single minimum (>= 3.25 enables the presets workflow the +standards call for) and align every `cmake_minimum_required` and the docs. + +### 5.3 CI does not run on pull requests, and skips its own quality gates — P1 + +**Problem.** The workflow triggers only on `push` to `main` +([.github/workflows/build.yml:3-5](.github/workflows/build.yml)). It builds and +tests, but never runs clang-tidy, a clang-format check, sanitizers, or coverage — +even though [cmake/Helpers.cmake](cmake/Helpers.cmake) provides all of them. + +**Impact.** Regressions and style/lint violations can merge unreviewed by CI; +PRs get no signal at all. + +**Fix.** Add a `pull_request` trigger and dedicated jobs for clang-format check, +clang-tidy, an ASAN/UBSAN test run, and coverage reporting. + +### 5.4 No fuzz target for the parser — P1 + +**Problem.** The project standards mandate fuzz testing for parsers on untrusted +input, but no fuzz target exists. + +**Impact.** The framing/dispatch path (especially before 1.1 lands) is the textbook +candidate for crashes and overreads on malformed input. + +**Fix.** Add a libFuzzer/AFL++ target that feeds arbitrary bytes through `parse`, +and run it in CI on a time budget. + +### 5.5 Typos in public CMake helper names — P2 + +**Problem.** Helper functions are misspelled: `anable_address_sanitizer` and +`build_documenation` ([cmake/Helpers.cmake:136,206](cmake/Helpers.cmake)). + +**Impact.** These are part of the build's public surface; renaming later is a +breaking change for anyone who depends on them. + +**Fix.** Rename to `enable_address_sanitizer` / `build_documentation` now, while the +blast radius is small. + +--- + +## 6. Testing + +### 6.1 Add golden / conformance tests against the official sample — P1 + +**Problem.** There is no end-to-end conformance check that a known input produces +known output. + +**Fix.** Parse a committed-or-fetched official sample and assert message counts per +type and spot-checked field values against a golden fixture. + +### 6.2 Broaden edge-case coverage — P1 + +**Problem.** Current tests should be extended to cover the failure and boundary +modes that matter for a parser and a book. + +**Fix.** Add cases for truncated buffers, zero-length frames, unknown type bytes, +endianness round-trips, and the full order-book lifecycle (add, execute, +execute-with-price, cancel, delete, replace, and crossed-book scenarios). + +--- + +## Priority summary + +| ID | Item | Priority | Area | +| --- | ------------------------------------------------- | -------- | ---------------- | +| 1.1 | Validate message length vs. expected size | P0 | Correctness | +| 1.2 | Remove `std::cerr` from the library | P0 | Correctness | +| 1.3 | Replace union type-punning byte swap | P0 | Correctness (UB) | +| 2.1 | Replace `std::map` + `std::function` dispatch | P0 | Performance | +| 2.2 | Fix or implement the zero-copy claim | P0 | Honesty | +| 2.3 | Allocation-free order book internals | P1 | Performance | +| 2.4 | Publish reproducible benchmarks | P1 | Performance | +| 3.1 | `std::span` buffer API | P1 | API | +| 3.2 | `std::expected` parse path | P1 | API | +| 3.3 | Typed fixed-point `Price` | P1 | API | +| 3.4 | Timestamp-to-wall-clock helpers | P1 | API | +| 5.1 | Remove committed large binaries / coverage report | P1 | Repo hygiene | +| 5.3 | CI on PRs + lint/sanitizer/coverage gates | P1 | CI | +| 5.4 | Add a parser fuzz target | P1 | Testing | +| 6.1 | Golden / conformance tests | P1 | Testing | +| 6.2 | Edge-case coverage | P1 | Testing | +| 4.1 | Symbol length >= 3 | P2 | Standards | +| 4.2 | `std::print`/`std::format` output | P2 | Standards | +| 4.3 | Frozen/`constexpr` indicator lookups | P2 | Standards | +| 5.2 | Single CMake minimum version | P2 | Build | +| 5.5 | Fix CMake helper name typos | P2 | Build | diff --git a/README.md b/README.md index 2e19d2e..664ecaa 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,10 @@ The design of this ITCH parser is guided by three principles: ## Key Features +- **Real Feed Ingestion**: Consume ITCH as it actually arrives — MoldUDP64 UDP + multicast framing, SoupBinTCP (Glimpse/recovery) framing, and `.pcap`/`.pcapng` + captures — with per-session sequence tracking and gap detection. No libpcap + dependency. See [Feed Ingestion](#feed-ingestion-transport). - **High Throughput**: Multi-gigabyte-per-second parsing on modern hardware (see [Benchmarks](#benchmarks) for measured numbers). - **Allocation-Free Core**: The callback-based parsing loop performs zero dynamic memory allocations, minimizing latency and jitter. - **Type-Safe API**: All ITCH messages are deserialized into a `std::variant` of dedicated, packed `struct`s, ensuring compile-time safety. @@ -442,6 +446,41 @@ The `print()` method provides a formatted snapshot of the top of the book. It di +------------+-------------+------------+-------------+ ``` +### Feed Ingestion (Transport) + +The raw parser assumes a concatenation of length-prefixed messages, but ITCH is +delivered inside transport framing. The `itch::transport` module decodes the +framing that actually arrives on the wire and on disk, then feeds the existing +parser. Everything is implemented in-house with **no libpcap dependency**. + +| Decoder | Header | Purpose | +| ------- | ------ | ------- | +| `MoldUdp64Decoder` | `itch/transport/moldudp64.hpp` | UDP multicast framing for live dissemination. | +| `SoupBinDecoder` | `itch/transport/soupbintcp.hpp` | TCP framing for Glimpse snapshots and recovery/replay. | +| `PcapReader` | `itch/transport/pcap.hpp` | Replay a feed from a `.pcap`/`.pcapng` capture file. | +| `SequenceTracker` | `itch/transport/sequencing.hpp` | Per-session sequence tracking, gap detection, recovery hooks. | + +```cpp +#include "itch/transport/pcap.hpp" + +itch::transport::PcapReader reader{[](const itch::Message& msg) { + // ... handle each decoded ITCH message ... +}}; + +// Surface any sequence gaps in the multicast stream. +reader.mold_decoder().tracker().set_gap_callback( + [](std::string_view session, std::uint64_t expected, std::uint64_t got) { + // ... drive recovery / log the gap ... + }); + +reader.read_file("capture.pcapng"); // walks Ethernet/IP/UDP/MoldUDP64 +// reader.messages_decoded(), reader.udp_datagrams(), tracker().gap_count() +``` + +For a live or captured MoldUDP64 datagram you already have in memory, call +`MoldUdp64Decoder::decode_packet(span)` directly; for a SoupBinTCP byte stream, +push segments through `SoupBinDecoder::feed(span)` as they arrive. + --- ## API Reference: ITCH 5.0 Message Types diff --git a/VERSION.txt b/VERSION.txt index 653f458..26aaba0 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1,2 +1 @@ -1.1.0 - +1.2.0 diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index a67a1fd..6066493 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -15,3 +15,11 @@ target_link_libraries( itch::itch warnings::strict ) + +add_executable(pcap_replay_example transport/pcap_replay_example.cpp) + +target_link_libraries( + pcap_replay_example PRIVATE + itch::itch + warnings::strict +) diff --git a/examples/transport/pcap_replay_example.cpp b/examples/transport/pcap_replay_example.cpp new file mode 100644 index 0000000..2bf6574 --- /dev/null +++ b/examples/transport/pcap_replay_example.cpp @@ -0,0 +1,50 @@ +#include +#include +#include + +#include "itch/messages.hpp" +#include "itch/transport/pcap.hpp" + +// Replays an ITCH feed straight from a captured .pcap/.pcapng file: the reader +// unwraps the Ethernet/IP/UDP and MoldUDP64 framing and yields decoded ITCH +// messages, with sequence gaps surfaced through the embedded tracker. +auto main(int argc, char* argv[]) -> int { + if (argc < 2) { + std::println(stderr, "Usage: {} [udp_port]", argv[0]); + return 1; + } + + std::uint64_t message_count = 0; + itch::transport::PcapReader reader {[&](const itch::Message& message) { + ++message_count; + const char type = std::visit([](const auto& msg) { return msg.message_type; }, message); + if (message_count <= 10) { + std::println("message {:>3}: type '{}'", message_count, type); + } + }}; + + reader.mold_decoder().tracker().set_gap_callback( + [](std::string_view session, std::uint64_t expected, std::uint64_t received) { + std::println( + stderr, "gap on session '{}': expected {} but got {}", session, expected, received + ); + } + ); + + if (argc >= 3) { + reader.set_udp_port_filter(static_cast(std::stoul(argv[2]))); + } + + if (!reader.read_file(argv[1])) { + std::println(stderr, "Error: '{}' is not a readable pcap/pcapng capture.", argv[1]); + return 1; + } + + std::println( + "Decoded {} ITCH messages from {} UDP datagrams ({} missing messages detected).", + reader.messages_decoded(), + reader.udp_datagrams(), + reader.mold_decoder().tracker().gap_count() + ); + return 0; +} diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index 2c1c9d0..3ead72a 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -17,3 +17,10 @@ target_compile_options(parser_fuzzer PRIVATE -g -fsanitize=fuzzer,address,undefi target_link_options(parser_fuzzer PRIVATE -fsanitize=fuzzer,address,undefined) message(STATUS "Added 'parser_fuzzer' target") + +add_executable(moldudp64_fuzzer moldudp64_fuzzer.cpp) +target_link_libraries(moldudp64_fuzzer PRIVATE itch::itch) +target_compile_options(moldudp64_fuzzer PRIVATE -g -fsanitize=fuzzer,address,undefined) +target_link_options(moldudp64_fuzzer PRIVATE -fsanitize=fuzzer,address,undefined) + +message(STATUS "Added 'moldudp64_fuzzer' target") diff --git a/fuzz/moldudp64_fuzzer.cpp b/fuzz/moldudp64_fuzzer.cpp new file mode 100644 index 0000000..642b608 --- /dev/null +++ b/fuzz/moldudp64_fuzzer.cpp @@ -0,0 +1,28 @@ +#include +#include +#include + +#include "itch/transport/moldudp64.hpp" +#include "itch/transport/pcap.hpp" +#include "itch/transport/soupbintcp.hpp" + +// libFuzzer entry point for the transport decoders, which sit in front of the +// parser on untrusted network input. Arbitrary bytes are pushed through the +// MoldUDP64, SoupBinTCP, and pcap paths; any crash, overread, or sanitizer +// report is a genuine bug. None of these paths throw on malformed input by +// design, so there is nothing to catch. +extern "C" auto LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size) -> int { + const auto* bytes = static_cast(static_cast(data)); + const std::span buffer {bytes, size}; + + itch::transport::MoldUdp64Decoder mold {[](const itch::Message&) noexcept {}}; + mold.decode_packet(buffer); + + itch::transport::SoupBinDecoder soup {[](const itch::Message&) noexcept {}}; + soup.feed(buffer); + + itch::transport::PcapReader pcap {[](const itch::Message&) noexcept {}}; + pcap.read(buffer); + + return 0; +} diff --git a/include/itch/transport/moldudp64.hpp b/include/itch/transport/moldudp64.hpp new file mode 100644 index 0000000..263e103 --- /dev/null +++ b/include/itch/transport/moldudp64.hpp @@ -0,0 +1,95 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "itch/parser.hpp" +#include "itch/transport/sequencing.hpp" + +namespace itch::transport { + +/// @brief The fixed 20-byte header that prefixes every MoldUDP64 downstream +/// packet. +/// +/// MoldUDP64 is the lightweight UDP multicast framing NASDAQ uses to disseminate +/// live market data. Each datagram carries a session identifier, the sequence +/// number of the first message it contains, and a count of the message blocks +/// that follow. The message blocks themselves are length-prefixed exactly like a +/// raw ITCH stream, so once the header is stripped the remainder feeds straight +/// into the ordinary `Parser` framing loop. +struct MoldUdp64Header { + std::array session {}; ///< Session id, space padded. + std::uint64_t sequence_number {0}; ///< Sequence of the first message block. + std::uint16_t message_count {0}; ///< Number of message blocks in the packet. + + /// @brief Sentinel `message_count` indicating an end-of-session packet. + static constexpr std::uint16_t END_OF_SESSION = 0xFFFF; + + /// @brief The session id as a view with trailing padding removed. + [[nodiscard]] auto session_view() const -> std::string_view { + std::size_t length = session.size(); + while (length > 0 && (session[length - 1] == ' ' || session[length - 1] == '\0')) { + --length; + } + return std::string_view {session.data(), length}; + } + + /// @brief A heartbeat carries no message blocks (`message_count == 0`). + [[nodiscard]] auto is_heartbeat() const noexcept -> bool { return message_count == 0; } + + /// @brief End-of-session is signalled by the sentinel `message_count`. + [[nodiscard]] auto is_end_of_session() const noexcept -> bool { + return message_count == END_OF_SESSION; + } +}; + +/// @brief Decodes MoldUDP64 datagrams and forwards the contained ITCH messages. +/// +/// One instance decodes a stream of datagrams from one or more sessions: call +/// `decode_packet` once per UDP payload. Each well-formed data packet's message +/// blocks are handed to an internal `Parser`, which invokes the supplied +/// `MessageCallback` for every decoded ITCH message. Per-packet sequence numbers +/// are fed to an embedded `SequenceTracker` so gaps in the multicast stream are +/// surfaced to the caller. +class MoldUdp64Decoder { + public: + /// @brief The on-wire size of the MoldUDP64 packet header, in bytes. + static constexpr std::size_t HEADER_SIZE = 20; + + /// @brief Constructs a decoder that calls `callback` for each ITCH message. + explicit MoldUdp64Decoder(MessageCallback callback); + + /// @brief Decodes a single MoldUDP64 datagram. + /// + /// @param packet The full UDP payload (header plus message blocks). + /// @return The parsed header, or `std::nullopt` if the datagram is too short + /// to contain a valid header. + auto decode_packet(std::span packet) -> std::optional; + + /// @brief The embedded sequence tracker (install gap callbacks here). + [[nodiscard]] auto tracker() noexcept -> SequenceTracker& { return m_tracker; } + [[nodiscard]] auto tracker() const noexcept -> const SequenceTracker& { return m_tracker; } + + /// @brief Total number of datagrams passed to `decode_packet`. + [[nodiscard]] auto packets_decoded() const noexcept -> std::uint64_t { + return m_packets_decoded; + } + + /// @brief Total number of ITCH messages decoded from all datagrams. + [[nodiscard]] auto messages_decoded() const noexcept -> std::uint64_t { + return m_messages_decoded; + } + + private: + Parser m_parser {}; + MessageCallback m_callback; + SequenceTracker m_tracker {}; + std::uint64_t m_packets_decoded {0}; + std::uint64_t m_messages_decoded {0}; +}; + +} // namespace itch::transport diff --git a/include/itch/transport/pcap.hpp b/include/itch/transport/pcap.hpp new file mode 100644 index 0000000..6b2f3d6 --- /dev/null +++ b/include/itch/transport/pcap.hpp @@ -0,0 +1,82 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "itch/parser.hpp" +#include "itch/transport/moldudp64.hpp" + +namespace itch::transport { + +/// @brief Replays ITCH market data straight from a captured network trace. +/// +/// Firms most often archive and share feeds as packet captures, so being able to +/// consume a `.pcap`/`.pcapng` file directly removes the need to pre-strip the +/// transport framing. The reader is implemented entirely in-house (no libpcap +/// dependency): it understands the classic pcap and the pcapng container formats, +/// walks the Ethernet / IPv4 / IPv6 / UDP layers of each captured frame, extracts +/// the UDP payload, and feeds it through an embedded `MoldUdp64Decoder`, which in +/// turn yields the decoded ITCH messages to the caller's callback. +/// +/// @note Only offline capture files are supported. Live capture would require a +/// platform packet-capture library and is tracked as future work behind a +/// separate build option. +class PcapReader { + public: + /// @brief Constructs a reader that forwards each decoded ITCH message to + /// `callback`. + explicit PcapReader(MessageCallback callback); + + /// @brief Decodes an in-memory capture buffer. + /// + /// @param capture The full contents of a `.pcap` or `.pcapng` file. + /// @return `true` if the buffer was a recognized capture format, `false` + /// otherwise (in which case nothing was decoded). + auto read(std::span capture) -> bool; + + /// @brief Reads and decodes a capture file from disk. + /// + /// @return `true` on a recognized, readable file; `false` if the file cannot + /// be opened or is not a capture. + auto read_file(const std::string& path) -> bool; + + /// @brief Restricts decoding to UDP datagrams sent to this destination port. + /// When unset, datagrams on any port are decoded. + auto set_udp_port_filter(std::uint16_t port) -> void { m_port_filter = port; } + + /// @brief Clears any destination-port filter previously installed. + auto clear_udp_port_filter() noexcept -> void { m_port_filter = std::nullopt; } + + /// @brief The embedded MoldUDP64 decoder (sequence tracking lives here). + [[nodiscard]] auto mold_decoder() noexcept -> MoldUdp64Decoder& { return m_mold; } + [[nodiscard]] auto mold_decoder() const noexcept -> const MoldUdp64Decoder& { return m_mold; } + + /// @brief Number of UDP datagrams extracted and handed to the MoldUDP64 + /// decoder. + [[nodiscard]] auto udp_datagrams() const noexcept -> std::uint64_t { return m_udp_datagrams; } + + /// @brief Total ITCH messages decoded across the whole capture. + [[nodiscard]] auto messages_decoded() const noexcept -> std::uint64_t { + return m_mold.messages_decoded(); + } + + private: + // Container-format readers; each returns false if the buffer is not that + // format or is structurally invalid. + auto read_classic_pcap(std::span capture, bool swapped) -> bool; + auto read_pcapng(std::span capture) -> bool; + + // Walks a single captured frame of the given link type and, if it carries a + // matching UDP datagram, feeds the payload to the MoldUDP64 decoder. + auto handle_frame(std::span frame, std::uint32_t link_type) -> void; + + MoldUdp64Decoder m_mold; + std::optional m_port_filter {}; + std::uint64_t m_udp_datagrams {0}; +}; + +} // namespace itch::transport diff --git a/include/itch/transport/sequencing.hpp b/include/itch/transport/sequencing.hpp new file mode 100644 index 0000000..afe2eb3 --- /dev/null +++ b/include/itch/transport/sequencing.hpp @@ -0,0 +1,130 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace itch::transport { + +/// @brief Abstract hook a caller implements to drive recovery against a replay +/// or retransmission service when a sequence gap is detected. +/// +/// The library never performs network I/O itself: when `SequenceTracker` sees a +/// gap it calls `request_retransmit`, and the caller is responsible for issuing +/// the actual re-request (for example a SoupBinTCP/MoldUDP64 rewind request) and +/// feeding the recovered messages back through the decoder. +class RetransmitRequester { + public: + RetransmitRequester() = default; + RetransmitRequester(const RetransmitRequester&) = default; + RetransmitRequester(RetransmitRequester&&) noexcept = default; + auto operator=(const RetransmitRequester&) -> RetransmitRequester& = default; + auto operator=(RetransmitRequester&&) noexcept -> RetransmitRequester& = default; + virtual ~RetransmitRequester() = default; + + /// @brief Requests retransmission of `count` messages starting at sequence + /// `start_sequence` for the given session. + virtual auto request_retransmit( + std::string_view session, std::uint64_t start_sequence, std::uint64_t count + ) -> void = 0; +}; + +/// @brief Tracks per-session sequence numbers across a transport layer and +/// surfaces gaps. +/// +/// Both the MoldUDP64 and SoupBinTCP decoders feed their per-packet sequence +/// information here. The tracker maintains, per session, the next sequence number +/// it expects to see. When an observed packet starts ahead of that expectation it +/// reports a gap (through the gap callback and the optional `RetransmitRequester`) +/// and resynchronizes. Duplicate or already-seen sequences are ignored so a +/// replayed recovery stream does not double-count. +class SequenceTracker { + public: + /// @brief Invoked once per detected gap with the session, the sequence the + /// tracker expected, and the (higher) sequence actually received. + using GapCallback = std::function< + void(std::string_view session, std::uint64_t expected, std::uint64_t received)>; + + /// @brief Records that a packet beginning at `first_sequence` carried `count` + /// sequenced messages for `session`. + /// + /// @return The number of missing messages detected (0 when in order). + auto observe(std::string_view session, std::uint64_t first_sequence, std::uint64_t count) + -> std::uint64_t { + const std::string key {session}; + auto iter = m_expected_by_session.find(key); + if (iter == m_expected_by_session.end()) { + // First time we see this session: anchor on its first sequence. + m_expected_by_session.emplace(key, first_sequence + count); + m_messages_seen += count; + return 0; + } + + std::uint64_t& expected = iter->second; + std::uint64_t gap = 0; + if (first_sequence > expected) { + gap = first_sequence - expected; + m_gap_count += gap; + if (m_gap_callback) { + m_gap_callback(session, expected, first_sequence); + } + if (m_requester != nullptr) { + m_requester->request_retransmit(session, expected, gap); + } + } + + const std::uint64_t end_sequence = first_sequence + count; + if (end_sequence > expected) { + // Count only the genuinely new messages, not replayed duplicates. + const std::uint64_t already_seen = + expected > first_sequence ? expected - first_sequence : 0; + m_messages_seen += count - std::min(count, already_seen); + expected = end_sequence; + } + return gap; + } + + /// @brief Installs the gap-notification callback (empty clears it). + auto set_gap_callback(GapCallback callback) -> void { m_gap_callback = std::move(callback); } + + /// @brief Installs a non-owning retransmission hook (nullptr clears it). + auto set_retransmit_requester(RetransmitRequester* requester) noexcept -> void { + m_requester = requester; + } + + /// @brief The next sequence number expected for a session, if it has been seen. + [[nodiscard]] auto expected_next(std::string_view session) const + -> std::optional { + const auto iter = m_expected_by_session.find(std::string {session}); + if (iter == m_expected_by_session.end()) { + return std::nullopt; + } + return iter->second; + } + + /// @brief Total number of missing messages detected across all sessions. + [[nodiscard]] auto gap_count() const noexcept -> std::uint64_t { return m_gap_count; } + + /// @brief Total number of distinct sequenced messages observed. + [[nodiscard]] auto messages_seen() const noexcept -> std::uint64_t { return m_messages_seen; } + + /// @brief Forgets all per-session state and resets the counters. + auto reset() -> void { + m_expected_by_session.clear(); + m_gap_count = 0; + m_messages_seen = 0; + } + + private: + std::unordered_map m_expected_by_session {}; + GapCallback m_gap_callback {}; + RetransmitRequester* m_requester {nullptr}; + std::uint64_t m_gap_count {0}; + std::uint64_t m_messages_seen {0}; +}; + +} // namespace itch::transport diff --git a/include/itch/transport/soupbintcp.hpp b/include/itch/transport/soupbintcp.hpp new file mode 100644 index 0000000..d20d552 --- /dev/null +++ b/include/itch/transport/soupbintcp.hpp @@ -0,0 +1,93 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "itch/parser.hpp" +#include "itch/transport/sequencing.hpp" + +namespace itch::transport { + +/// @brief The SoupBinTCP packet types (the one-byte type that follows the +/// 2-byte length prefix). +/// +/// SoupBinTCP is the reliable, ordered TCP framing NASDAQ uses for the Glimpse +/// snapshot service and for recovery/replay. Only the server-to-client subset is +/// needed to consume a captured or replayed stream, but every defined type is +/// listed for completeness. +enum class SoupBinPacketType : char { + debug = '+', ///< Free-form debug text (either direction). + login_accepted = 'A', ///< Session id (10) + starting sequence number (20, ASCII). + login_rejected = 'J', ///< Reject reason code (1). + sequenced_data = 'S', ///< One sequenced application (ITCH) message. + server_heartbeat = 'H', ///< Keep-alive from the server. + end_of_session = 'Z', ///< The server has finished the session. + login_request = 'L', ///< Client login request. + unsequenced_data = 'U', ///< One unsequenced application message. + client_heartbeat = 'R', ///< Keep-alive from the client. + logout_request = 'O', ///< Client logout request. +}; + +/// @brief A stateful decoder for a SoupBinTCP byte stream. +/// +/// Because TCP delivers a byte stream rather than discrete packets, the decoder +/// accumulates input across `feed` calls and emits messages only for complete +/// packets. Sequenced (and, optionally, unsequenced) data packets each carry a +/// single ITCH message, which is decoded through an internal `Parser` and handed +/// to the `MessageCallback`. Control packets (login, heartbeat, logout, end of +/// session) are surfaced through the optional event callback. The starting +/// sequence number is learned from the Login Accepted packet and advanced per +/// sequenced message, with gaps reported through the embedded `SequenceTracker`. +class SoupBinDecoder { + public: + /// @brief Invoked for each non-data control packet, with its raw payload. + using EventCallback = + std::function payload)>; + + /// @brief Constructs a decoder that calls `callback` for each ITCH message. + explicit SoupBinDecoder(MessageCallback callback); + + /// @brief Feeds a chunk of the TCP byte stream, processing any complete + /// packets it completes. + auto feed(std::span bytes) -> void; + + /// @brief Installs the control-packet event callback (empty clears it). + auto set_event_callback(EventCallback callback) -> void; + + /// @brief The embedded sequence tracker (install gap callbacks here). + [[nodiscard]] auto tracker() noexcept -> SequenceTracker& { return m_tracker; } + [[nodiscard]] auto tracker() const noexcept -> const SequenceTracker& { return m_tracker; } + + /// @brief The session id learned from Login Accepted (empty until then). + [[nodiscard]] auto current_session() const -> std::string_view { return m_session; } + + /// @brief The next sequence number the decoder expects for a data packet. + [[nodiscard]] auto next_sequence() const noexcept -> std::uint64_t { return m_next_sequence; } + + /// @brief Total sequenced/unsequenced data messages decoded. + [[nodiscard]] auto messages_decoded() const noexcept -> std::uint64_t { + return m_messages_decoded; + } + + private: + // Decodes the single packet [type byte + payload] and dispatches it. + auto process_packet(SoupBinPacketType type, std::span payload) -> void; + // Decodes one application message payload through the parser. + auto decode_application_message(std::span payload) -> void; + + Parser m_parser {}; + MessageCallback m_callback; + EventCallback m_event_callback {}; + SequenceTracker m_tracker {}; + std::vector m_buffer {}; + std::string m_session {}; + std::uint64_t m_next_sequence {1}; + std::uint64_t m_messages_decoded {0}; +}; + +} // namespace itch::transport diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d5da1a3..3adcc31 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,10 +4,13 @@ include(GNUInstallDirs) # Step 1: Add targets and properties add_library( - itch - parser.cpp - messages.cpp + itch + parser.cpp + messages.cpp order_book.cpp + transport/moldudp64.cpp + transport/soupbintcp.cpp + transport/pcap.cpp ) add_library(itch::itch ALIAS itch) if (NOT ANDROID) diff --git a/src/transport/moldudp64.cpp b/src/transport/moldudp64.cpp new file mode 100644 index 0000000..78f6baa --- /dev/null +++ b/src/transport/moldudp64.cpp @@ -0,0 +1,72 @@ +#include "itch/transport/moldudp64.hpp" + +#include +#include +#include + +namespace itch::transport { + +namespace { + +// Reads a big-endian unsigned integer of the field's width out of a byte span. +template +auto read_big_endian(std::span data, std::size_t offset) -> UintType { + UintType value {}; + std::memcpy(&value, data.data() + offset, sizeof(UintType)); + return utils::from_big_endian(value); +} + +} // namespace + +MoldUdp64Decoder::MoldUdp64Decoder(MessageCallback callback) : m_callback {std::move(callback)} {} + +auto MoldUdp64Decoder::decode_packet(std::span packet) + -> std::optional { + if (packet.size() < HEADER_SIZE) { + return std::nullopt; + } + + ++m_packets_decoded; + + MoldUdp64Header header {}; + std::memcpy(header.session.data(), packet.data(), header.session.size()); + constexpr std::size_t SEQUENCE_OFFSET = 10; + constexpr std::size_t COUNT_OFFSET = 18; + header.sequence_number = read_big_endian(packet, SEQUENCE_OFFSET); + header.message_count = read_big_endian(packet, COUNT_OFFSET); + + if (header.is_end_of_session()) { + return header; // Control packet: no message blocks, no sequence advance. + } + + // The sequence number names the first message in the packet; feed the count + // to the tracker so gaps in the multicast stream are detected. A heartbeat + // (count 0) still anchors the next expected sequence. + m_tracker.observe(header.session_view(), header.sequence_number, header.message_count); + + if (header.is_heartbeat()) { + return header; + } + + // After the header the datagram is a sequence of length-prefixed message + // blocks, which is exactly the framing the ordinary parser consumes. A + // malformed or truncated datagram is tolerated: the parser stops at the end + // of the buffer and we report through the callback what was decoded. + const std::span blocks = packet.subspan(HEADER_SIZE); + auto counting_callback = [this](const Message& message) { + ++m_messages_decoded; + if (m_callback) { + m_callback(message); + } + }; + try { + m_parser.parse(blocks, counting_callback); + } catch (const std::runtime_error&) { + // A truncated trailing block in a single datagram is non-fatal here; the + // gap will already have been surfaced by the sequence tracker. + } + + return header; +} + +} // namespace itch::transport diff --git a/src/transport/pcap.cpp b/src/transport/pcap.cpp new file mode 100644 index 0000000..3f622ef --- /dev/null +++ b/src/transport/pcap.cpp @@ -0,0 +1,255 @@ +#include "itch/transport/pcap.hpp" + +#include +#include +#include +#include + +namespace itch::transport { + +namespace { + +// Capture-file record fields follow the file's own byte order; network protocol +// headers (Ethernet/IP/UDP) are always big-endian. These helpers read either. +template +auto read_le_or_be(std::span data, std::size_t offset, bool swapped) -> UintType { + UintType value {}; + std::memcpy(&value, data.data() + offset, sizeof(UintType)); + return swapped ? utils::swap_bytes(value) : value; +} + +template +auto read_net(std::span data, std::size_t offset) -> UintType { + UintType value {}; + std::memcpy(&value, data.data() + offset, sizeof(UintType)); + return utils::from_big_endian(value); +} + +// Link-layer types we know how to unwrap (subset of the tcpdump/pcap registry). +constexpr std::uint32_t LINKTYPE_ETHERNET = 1; +constexpr std::uint32_t LINKTYPE_RAW = 101; +constexpr std::uint32_t LINKTYPE_LINUX_SLL = 113; + +constexpr std::uint16_t ETHERTYPE_IPV4 = 0x0800; +constexpr std::uint16_t ETHERTYPE_IPV6 = 0x86DD; +constexpr std::uint16_t ETHERTYPE_VLAN = 0x8100; + +constexpr std::uint8_t IP_PROTOCOL_UDP = 17; + +constexpr std::size_t ETHERNET_HEADER_SIZE = 14; +constexpr std::size_t VLAN_TAG_SIZE = 4; +constexpr std::size_t LINUX_SLL_HEADER_SIZE = 16; +constexpr std::size_t IPV6_HEADER_SIZE = 40; +constexpr std::size_t UDP_HEADER_SIZE = 8; + +constexpr std::uint32_t PCAP_MAGIC = 0xA1B2C3D4; +constexpr std::uint32_t PCAP_MAGIC_SWAPPED = 0xD4C3B2A1; +constexpr std::uint32_t PCAP_MAGIC_NANO = 0xA1B23C4D; +constexpr std::uint32_t PCAP_MAGIC_NANO_SWP = 0x4D3CB2A1; +constexpr std::uint32_t PCAPNG_BLOCK_SHB = 0x0A0D0D0A; + +} // namespace + +PcapReader::PcapReader(MessageCallback callback) : m_mold {std::move(callback)} {} + +auto PcapReader::read(std::span capture) -> bool { + if (capture.size() < sizeof(std::uint32_t)) { + return false; + } + std::uint32_t magic {}; + std::memcpy(&magic, capture.data(), sizeof(magic)); + + if (magic == PCAP_MAGIC || magic == PCAP_MAGIC_NANO) { + return read_classic_pcap(capture, /*swapped=*/false); + } + if (magic == PCAP_MAGIC_SWAPPED || magic == PCAP_MAGIC_NANO_SWP) { + return read_classic_pcap(capture, /*swapped=*/true); + } + if (magic == PCAPNG_BLOCK_SHB) { + return read_pcapng(capture); + } + return false; +} + +auto PcapReader::read_file(const std::string& path) -> bool { + std::ifstream file {path, std::ios::binary}; + if (!file) { + return false; + } + std::vector raw {std::istreambuf_iterator {file}, {}}; + std::vector bytes(raw.size()); + std::memcpy(bytes.data(), raw.data(), raw.size()); + return read(std::span {bytes}); +} + +auto PcapReader::read_classic_pcap(std::span capture, bool swapped) -> bool { + constexpr std::size_t GLOBAL_HEADER_SIZE = 24; + constexpr std::size_t NETWORK_OFFSET = 20; + constexpr std::size_t RECORD_HEADER_SIZE = 16; + constexpr std::size_t INCL_LEN_OFFSET = 8; + + if (capture.size() < GLOBAL_HEADER_SIZE) { + return false; + } + const auto link_type = read_le_or_be(capture, NETWORK_OFFSET, swapped); + + std::size_t offset = GLOBAL_HEADER_SIZE; + while (offset + RECORD_HEADER_SIZE <= capture.size()) { + const auto incl_len = + read_le_or_be(capture, offset + INCL_LEN_OFFSET, swapped); + offset += RECORD_HEADER_SIZE; + if (offset + incl_len > capture.size()) { + break; // Truncated final record. + } + handle_frame(capture.subspan(offset, incl_len), link_type); + offset += incl_len; + } + return true; +} + +auto PcapReader::read_pcapng(std::span capture) -> bool { + constexpr std::size_t BLOCK_HEADER_SIZE = 8; // Type + total length. + constexpr std::size_t BOM_OFFSET = 8; // Byte-order magic in the SHB. + constexpr std::uint32_t BOM_LITTLE = 0x1A2B3C4D; + constexpr std::uint32_t BLOCK_IDB = 0x00000001; + constexpr std::uint32_t BLOCK_SPB = 0x00000003; + constexpr std::uint32_t BLOCK_EPB = 0x00000006; + + bool swapped = false; + std::vector interface_link_types; + + std::size_t offset = 0; + while (offset + BLOCK_HEADER_SIZE <= capture.size()) { + std::uint32_t block_type {}; + std::memcpy(&block_type, capture.data() + offset, sizeof(block_type)); + // The SHB's byte-order magic tells us how to read this section. Until we + // have read it, block lengths are interpreted in host order, which is + // correct for the SHB type/length fields we need to bootstrap from. + const bool block_swapped = (block_type == PCAPNG_BLOCK_SHB) ? false : swapped; + + const auto total_length = + read_le_or_be(capture, offset + sizeof(std::uint32_t), block_swapped); + if (total_length < BLOCK_HEADER_SIZE + sizeof(std::uint32_t) || + offset + total_length > capture.size()) { + break; + } + + if (block_type == PCAPNG_BLOCK_SHB) { + const auto bom = read_net(capture, offset + BOM_OFFSET); + // If the byte-order magic reads as the canonical value in network + // (big-endian) interpretation, the file is big-endian; otherwise it + // is little-endian. Compare against both forms. + std::uint32_t bom_native {}; + std::memcpy(&bom_native, capture.data() + offset + BOM_OFFSET, sizeof(bom_native)); + swapped = (bom_native != BOM_LITTLE); + (void)bom; + interface_link_types.clear(); + } else if (block_type == BLOCK_IDB) { + // LinkType is the first 2-byte field of the IDB body. + const auto link_type = + read_le_or_be(capture, offset + BLOCK_HEADER_SIZE, swapped); + interface_link_types.push_back(link_type); + } else if (block_type == BLOCK_EPB) { + constexpr std::size_t EPB_IFACE_OFFSET = BLOCK_HEADER_SIZE; + constexpr std::size_t EPB_CAPLEN_OFFSET = BLOCK_HEADER_SIZE + 12; + constexpr std::size_t EPB_DATA_OFFSET = BLOCK_HEADER_SIZE + 20; + const auto iface = + read_le_or_be(capture, offset + EPB_IFACE_OFFSET, swapped); + const auto cap_len = + read_le_or_be(capture, offset + EPB_CAPLEN_OFFSET, swapped); + if (offset + EPB_DATA_OFFSET + cap_len <= capture.size() && + iface < interface_link_types.size()) { + handle_frame( + capture.subspan(offset + EPB_DATA_OFFSET, cap_len), interface_link_types[iface] + ); + } + } else if (block_type == BLOCK_SPB) { + constexpr std::size_t SPB_DATA_OFFSET = BLOCK_HEADER_SIZE + 4; + // The SPB has no captured length, so derive it from the block size. + if (total_length >= SPB_DATA_OFFSET + sizeof(std::uint32_t) && + !interface_link_types.empty()) { + const std::size_t data_len = total_length - SPB_DATA_OFFSET - sizeof(std::uint32_t); + handle_frame( + capture.subspan(offset + SPB_DATA_OFFSET, data_len), + interface_link_types.front() + ); + } + } + + offset += total_length; + } + return true; +} + +auto PcapReader::handle_frame(std::span frame, std::uint32_t link_type) -> void { + // Strip the link layer down to the network (IP) header. + std::span network = frame; + if (link_type == LINKTYPE_ETHERNET) { + if (frame.size() < ETHERNET_HEADER_SIZE) { + return; + } + std::size_t header = ETHERNET_HEADER_SIZE; + std::uint16_t ethertype = read_net(frame, 12); + while (ethertype == ETHERTYPE_VLAN && frame.size() >= header + VLAN_TAG_SIZE) { + ethertype = read_net(frame, header + 2); + header += VLAN_TAG_SIZE; + } + if (ethertype != ETHERTYPE_IPV4 && ethertype != ETHERTYPE_IPV6) { + return; + } + network = frame.subspan(header); + } else if (link_type == LINKTYPE_LINUX_SLL) { + if (frame.size() < LINUX_SLL_HEADER_SIZE) { + return; + } + network = frame.subspan(LINUX_SLL_HEADER_SIZE); + } else if (link_type != LINKTYPE_RAW) { + return; // Unsupported link layer. + } + + if (network.empty()) { + return; + } + + // Determine IP version and locate the UDP header. + const auto version = static_cast(network[0]) >> 4; + std::uint8_t protocol {0}; + std::size_t ip_header_len {0}; + if (version == 4) { + ip_header_len = (static_cast(network[0]) & 0x0F) * 4U; + if (network.size() < ip_header_len || ip_header_len < 20) { + return; + } + protocol = static_cast(network[9]); + } else if (version == 6) { + if (network.size() < IPV6_HEADER_SIZE) { + return; + } + protocol = static_cast(network[6]); + ip_header_len = IPV6_HEADER_SIZE; + } else { + return; + } + if (protocol != IP_PROTOCOL_UDP) { + return; + } + + const std::span udp = network.subspan(ip_header_len); + if (udp.size() < UDP_HEADER_SIZE) { + return; + } + const auto dst_port = read_net(udp, 2); + if (m_port_filter.has_value() && dst_port != *m_port_filter) { + return; + } + auto udp_length = read_net(udp, 4); + std::size_t payload_len = udp.size() - UDP_HEADER_SIZE; + if (udp_length >= UDP_HEADER_SIZE) { + payload_len = std::min(payload_len, udp_length - UDP_HEADER_SIZE); + } + + ++m_udp_datagrams; + m_mold.decode_packet(udp.subspan(UDP_HEADER_SIZE, payload_len)); +} + +} // namespace itch::transport diff --git a/src/transport/soupbintcp.cpp b/src/transport/soupbintcp.cpp new file mode 100644 index 0000000..87d262b --- /dev/null +++ b/src/transport/soupbintcp.cpp @@ -0,0 +1,157 @@ +#include "itch/transport/soupbintcp.hpp" + +#include +#include +#include +#include +#include +#include + +namespace itch::transport { + +namespace { + +constexpr std::size_t LENGTH_PREFIX_SIZE = 2; +constexpr std::size_t LOGIN_SESSION_SIZE = 10; +constexpr std::size_t LOGIN_SEQUENCE_SIZE = 20; +constexpr std::size_t MAX_WIRE_MESSAGE_LEN = 0xFFFF; + +// Trims trailing spaces from a fixed-width ASCII field. +auto trim_field(std::span field) -> std::string { + std::size_t length = field.size(); + while (length > 0 && static_cast(field[length - 1]) == ' ') { + --length; + } + std::string result(length, '\0'); + for (std::size_t index = 0; index < length; ++index) { + result[index] = static_cast(field[index]); + } + return result; +} + +// Parses the right-justified numeric sequence number field of Login Accepted. +auto parse_sequence_field(std::span field) -> std::uint64_t { + const std::string text = trim_field(field); + std::uint64_t value = 0; + const char* begin = text.data(); + // Skip any leading spaces left after trimming only trailing ones. + while (begin != text.data() + text.size() && *begin == ' ') { + ++begin; + } + std::from_chars(begin, text.data() + text.size(), value); + return value; +} + +} // namespace + +SoupBinDecoder::SoupBinDecoder(MessageCallback callback) : m_callback {std::move(callback)} {} + +auto SoupBinDecoder::set_event_callback(EventCallback callback) -> void { + m_event_callback = std::move(callback); +} + +auto SoupBinDecoder::feed(std::span bytes) -> void { + m_buffer.insert(m_buffer.end(), bytes.begin(), bytes.end()); + + std::size_t offset = 0; + while (m_buffer.size() - offset >= LENGTH_PREFIX_SIZE) { + std::uint16_t packet_length {}; + std::memcpy(&packet_length, m_buffer.data() + offset, LENGTH_PREFIX_SIZE); + packet_length = utils::from_big_endian(packet_length); + + if (packet_length == 0) { + offset += LENGTH_PREFIX_SIZE; // Defensive: skip a zero-length frame. + continue; + } + if (m_buffer.size() - offset < LENGTH_PREFIX_SIZE + packet_length) { + break; // The rest of this packet has not arrived yet. + } + + const auto type = static_cast( + static_cast(m_buffer[offset + LENGTH_PREFIX_SIZE]) + ); + const std::span payload { + m_buffer.data() + offset + LENGTH_PREFIX_SIZE + 1, + static_cast(packet_length) - 1 + }; + process_packet(type, payload); + offset += LENGTH_PREFIX_SIZE + packet_length; + } + + // Drop the bytes we have fully consumed, keeping any partial trailing packet. + if (offset > 0) { + m_buffer.erase(m_buffer.begin(), m_buffer.begin() + static_cast(offset)); + } +} + +auto SoupBinDecoder::process_packet(SoupBinPacketType type, std::span payload) + -> void { + switch (type) { + case SoupBinPacketType::login_accepted: { + if (payload.size() >= LOGIN_SESSION_SIZE + LOGIN_SEQUENCE_SIZE) { + m_session = trim_field(payload.subspan(0, LOGIN_SESSION_SIZE)); + m_next_sequence = + parse_sequence_field(payload.subspan(LOGIN_SESSION_SIZE, LOGIN_SEQUENCE_SIZE)); + if (m_next_sequence == 0) { + m_next_sequence = 1; + } + } + break; + } + case SoupBinPacketType::sequenced_data: { + m_tracker.observe(m_session, m_next_sequence, 1); + ++m_next_sequence; + decode_application_message(payload); + return; + } + case SoupBinPacketType::unsequenced_data: { + decode_application_message(payload); + return; + } + case SoupBinPacketType::debug: + case SoupBinPacketType::login_rejected: + case SoupBinPacketType::server_heartbeat: + case SoupBinPacketType::end_of_session: + case SoupBinPacketType::login_request: + case SoupBinPacketType::client_heartbeat: + case SoupBinPacketType::logout_request: + break; + } + + if (m_event_callback) { + m_event_callback(type, payload); + } +} + +auto SoupBinDecoder::decode_application_message(std::span payload) -> void { + if (payload.empty() || payload.size() > MAX_WIRE_MESSAGE_LEN) { + return; + } + + // A data packet carries one ITCH message without its own 2-byte length + // prefix; re-prefixing it lets the ordinary framing parser decode it. The + // length is written big-endian (network order) to match the wire framing. + std::vector framed; + framed.reserve(LENGTH_PREFIX_SIZE + payload.size()); + const auto length = static_cast(payload.size()); + const auto big_endian = utils::from_big_endian(length); + const auto len_bytes = std::bit_cast>(big_endian); + framed.push_back(len_bytes[0]); + framed.push_back(len_bytes[1]); + framed.insert(framed.end(), payload.begin(), payload.end()); + + auto counting_callback = [this](const Message& message) { + ++m_messages_decoded; + if (m_callback) { + m_callback(message); + } + }; + try { + m_parser.parse(std::span {framed}, counting_callback); + } catch (const std::runtime_error&) { + // A malformed single-message payload is skipped rather than aborting the + // whole stream. + } +} + +} // namespace itch::transport diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 95c9d30..dcd452a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -22,8 +22,11 @@ add_executable( test_order_book_extra.cpp test_price_time.cpp test_conformance.cpp + transport/test_transport.cpp ) +target_include_directories(itch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries( itch_tests PRIVATE itch::itch diff --git a/tests/transport/frame_builders.hpp b/tests/transport/frame_builders.hpp new file mode 100644 index 0000000..2e20bdc --- /dev/null +++ b/tests/transport/frame_builders.hpp @@ -0,0 +1,163 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "itch/parser.hpp" + +// Shared helpers for the transport tests: build raw ITCH frames and wrap them in +// the MoldUDP64 / SoupBinTCP / pcap envelopes from in-memory bytes, so the +// decoders can be exercised without any real capture files. +namespace itch::test { + +inline auto append_be16(std::vector& out, std::uint16_t value) -> void { + const auto big = itch::utils::from_big_endian(value); + std::array bytes {}; + std::memcpy(bytes.data(), &big, sizeof(big)); + out.insert(out.end(), bytes.begin(), bytes.end()); +} + +inline auto append_be32(std::vector& out, std::uint32_t value) -> void { + const auto big = itch::utils::from_big_endian(value); + std::array bytes {}; + std::memcpy(bytes.data(), &big, sizeof(big)); + out.insert(out.end(), bytes.begin(), bytes.end()); +} + +inline auto append_be64(std::vector& out, std::uint64_t value) -> void { + const auto big = itch::utils::from_big_endian(value); + std::array bytes {}; + std::memcpy(bytes.data(), &big, sizeof(big)); + out.insert(out.end(), bytes.begin(), bytes.end()); +} + +inline auto append_le32(std::vector& out, std::uint32_t value) -> void { + std::array bytes {}; + std::memcpy(bytes.data(), &value, sizeof(value)); + out.insert(out.end(), bytes.begin(), bytes.end()); +} + +inline auto append_bytes(std::vector& out, const std::vector& src) -> void { + out.insert(out.end(), src.begin(), src.end()); +} + +/// @brief Builds the raw payload (no length prefix) of a System Event message. +inline auto system_event_payload(std::uint64_t timestamp, char event_code) + -> std::vector { + std::vector payload; + payload.push_back(std::byte {'S'}); + append_be16(payload, 1); // stock_locate + append_be16(payload, 0); // tracking_number + // 48-bit timestamp: take the low 6 bytes of a big-endian 64-bit value. + std::vector ts; + append_be64(ts, timestamp); + payload.insert(payload.end(), ts.begin() + 2, ts.end()); + payload.push_back(std::byte {static_cast(event_code)}); + return payload; +} + +/// @brief Wraps a raw payload as a length-prefixed ITCH frame (and message +/// block, since both use the same 2-byte big-endian length prefix). +inline auto length_prefixed(const std::vector& payload) -> std::vector { + std::vector frame; + append_be16(frame, static_cast(payload.size())); + frame.insert(frame.end(), payload.begin(), payload.end()); + return frame; +} + +/// @brief Builds a complete MoldUDP64 datagram from a list of raw message +/// payloads. +inline auto moldudp64_packet( + const std::string& session, + std::uint64_t sequence, + const std::vector>& payloads +) -> std::vector { + std::vector packet; + std::array session_field {}; + for (std::size_t index = 0; index < session_field.size(); ++index) { + session_field[index] = index < session.size() ? session[index] : ' '; + } + for (char chr : session_field) { + packet.push_back(std::byte {static_cast(chr)}); + } + append_be64(packet, sequence); + append_be16(packet, static_cast(payloads.size())); + for (const auto& payload : payloads) { + append_bytes(packet, length_prefixed(payload)); + } + return packet; +} + +/// @brief Builds a SoupBinTCP packet (2-byte length + type byte + payload). +inline auto soupbin_packet(char type, const std::vector& payload) + -> std::vector { + std::vector packet; + append_be16(packet, static_cast(payload.size() + 1)); + packet.push_back(std::byte {static_cast(type)}); + packet.insert(packet.end(), payload.begin(), payload.end()); + return packet; +} + +/// @brief Wraps a UDP payload in Ethernet/IPv4/UDP headers (a single frame). +inline auto ethernet_ipv4_udp_frame(std::uint16_t dst_port, const std::vector& payload) + -> std::vector { + std::vector frame; + // Ethernet header: dst(6), src(6), ethertype(2). + for (int index = 0; index < 12; ++index) { + frame.push_back(std::byte {0}); + } + append_be16(frame, 0x0800); // IPv4 + + const std::uint16_t udp_len = static_cast(8 + payload.size()); + const std::uint16_t ip_len = static_cast(20 + udp_len); + + // IPv4 header (20 bytes, no options). + frame.push_back(std::byte {0x45}); // version 4, IHL 5 + frame.push_back(std::byte {0}); // DSCP/ECN + append_be16(frame, ip_len); // total length + append_be16(frame, 0); // identification + append_be16(frame, 0); // flags + fragment offset + frame.push_back(std::byte {64}); // TTL + frame.push_back(std::byte {17}); // protocol = UDP + append_be16(frame, 0); // header checksum (ignored) + append_be32(frame, 0x7F000001); // src ip + append_be32(frame, 0x7F000001); // dst ip + + // UDP header (8 bytes). + append_be16(frame, 12345); // src port + append_be16(frame, dst_port); // dst port + append_be16(frame, udp_len); // length + append_be16(frame, 0); // checksum (ignored) + + frame.insert(frame.end(), payload.begin(), payload.end()); + return frame; +} + +/// @brief Builds a classic little-endian .pcap file from a list of frames. +inline auto classic_pcap(const std::vector>& frames) + -> std::vector { + std::vector file; + append_le32(file, 0xA1B2C3D4); // magic + file.push_back(std::byte {2}); // version major (LE) + file.push_back(std::byte {0}); + file.push_back(std::byte {4}); // version minor + file.push_back(std::byte {0}); + append_le32(file, 0); // thiszone + append_le32(file, 0); // sigfigs + append_le32(file, 65535); // snaplen + append_le32(file, 1); // network = Ethernet + + for (const auto& frame : frames) { + append_le32(file, 0); // ts_sec + append_le32(file, 0); // ts_usec + append_le32(file, static_cast(frame.size())); // incl_len + append_le32(file, static_cast(frame.size())); // orig_len + file.insert(file.end(), frame.begin(), frame.end()); + } + return file; +} + +} // namespace itch::test diff --git a/tests/transport/test_transport.cpp b/tests/transport/test_transport.cpp new file mode 100644 index 0000000..d518b28 --- /dev/null +++ b/tests/transport/test_transport.cpp @@ -0,0 +1,183 @@ +#include + +#include +#include + +#include "itch/messages.hpp" +#include "itch/transport/moldudp64.hpp" +#include "itch/transport/pcap.hpp" +#include "itch/transport/sequencing.hpp" +#include "itch/transport/soupbintcp.hpp" +#include "transport/frame_builders.hpp" + +namespace { + +using itch::Message; +using itch::SystemEventMessage; + +// Pulls the event_code out of the System Event messages decoded into a vector. +auto event_codes(const std::vector& messages) -> std::string { + std::string codes; + for (const auto& message : messages) { + if (const auto* event = std::get_if(&message)) { + codes.push_back(event->event_code); + } + } + return codes; +} + +} // namespace + +TEST(MoldUdp64, DecodesAllMessageBlocksInPacket) { + std::vector decoded; + itch::transport::MoldUdp64Decoder decoder {[&](const Message& msg) { decoded.push_back(msg); }}; + + const auto packet = itch::test::moldudp64_packet( + "SESSION01", + 1, + {itch::test::system_event_payload(1000, 'O'), itch::test::system_event_payload(2000, 'S')} + ); + + const auto header = decoder.decode_packet(std::span {packet}); + ASSERT_TRUE(header.has_value()); + EXPECT_EQ(header->session_view(), "SESSION01"); + EXPECT_EQ(header->sequence_number, 1U); + EXPECT_EQ(header->message_count, 2U); + EXPECT_EQ(decoder.messages_decoded(), 2U); + EXPECT_EQ(event_codes(decoded), "OS"); +} + +TEST(MoldUdp64, HeartbeatCarriesNoMessages) { + itch::transport::MoldUdp64Decoder decoder {[](const Message&) {}}; + const auto packet = itch::test::moldudp64_packet("SESSION01", 1, {}); + const auto header = decoder.decode_packet(std::span {packet}); + ASSERT_TRUE(header.has_value()); + EXPECT_TRUE(header->is_heartbeat()); + EXPECT_EQ(decoder.messages_decoded(), 0U); +} + +TEST(MoldUdp64, DetectsSequenceGapAcrossPackets) { + itch::transport::MoldUdp64Decoder decoder {[](const Message&) {}}; + std::uint64_t reported_expected = 0; + std::uint64_t reported_received = 0; + decoder.tracker().set_gap_callback( + [&](std::string_view, std::uint64_t expected, std::uint64_t received) { + reported_expected = expected; + reported_received = received; + } + ); + + // Sequences 1 and 2, then jump to 5 (missing 3 and 4). + const auto first = itch::test::moldudp64_packet( + "S", 1, {itch::test::system_event_payload(1, 'O'), itch::test::system_event_payload(2, 'S')} + ); + const auto second = + itch::test::moldudp64_packet("S", 5, {itch::test::system_event_payload(3, 'Q')}); + decoder.decode_packet(std::span {first}); + decoder.decode_packet(std::span {second}); + + EXPECT_EQ(decoder.tracker().gap_count(), 2U); + EXPECT_EQ(reported_expected, 3U); + EXPECT_EQ(reported_received, 5U); +} + +TEST(MoldUdp64, ShortDatagramReturnsNullopt) { + itch::transport::MoldUdp64Decoder decoder {[](const Message&) {}}; + std::vector tiny(4, std::byte {0}); + EXPECT_FALSE(decoder.decode_packet(std::span {tiny}).has_value()); +} + +TEST(SoupBinTcp, DecodesSequencedDataAcrossSegmentBoundaries) { + std::vector decoded; + itch::transport::SoupBinDecoder decoder {[&](const Message& msg) { decoded.push_back(msg); }}; + + const auto packet1 = itch::test::soupbin_packet('S', itch::test::system_event_payload(10, 'O')); + const auto packet2 = itch::test::soupbin_packet('S', itch::test::system_event_payload(20, 'Q')); + + // Concatenate and feed in two chunks that split a packet in half. + std::vector stream; + stream.insert(stream.end(), packet1.begin(), packet1.end()); + stream.insert(stream.end(), packet2.begin(), packet2.end()); + const std::size_t split = packet1.size() + 2; + decoder.feed(std::span {stream.data(), split}); + EXPECT_EQ(decoded.size(), 1U); // Second packet is still incomplete. + decoder.feed(std::span {stream.data() + split, stream.size() - split}); + + EXPECT_EQ(decoder.messages_decoded(), 2U); + EXPECT_EQ(event_codes(decoded), "OQ"); +} + +TEST(SoupBinTcp, LoginAcceptedSetsSessionAndSequence) { + itch::transport::SoupBinDecoder decoder {[](const Message&) {}}; + + std::vector login; + const std::string session = "GLIMPSE "; // 10 chars + for (char chr : session) { + login.push_back(std::byte {static_cast(chr)}); + } + const std::string sequence = " 42"; // 20 chars, right-justified + for (char chr : sequence) { + login.push_back(std::byte {static_cast(chr)}); + } + const auto packet = itch::test::soupbin_packet('A', login); + decoder.feed(std::span {packet}); + + EXPECT_EQ(decoder.current_session(), "GLIMPSE"); + EXPECT_EQ(decoder.next_sequence(), 42U); +} + +TEST(SoupBinTcp, ControlPacketsReachEventCallback) { + itch::transport::SoupBinDecoder decoder {[](const Message&) {}}; + char seen = '\0'; + decoder.set_event_callback([&](itch::transport::SoupBinPacketType type, + std::span) { seen = static_cast(type); }); + const auto heartbeat = itch::test::soupbin_packet('H', {}); + decoder.feed(std::span {heartbeat}); + EXPECT_EQ(seen, 'H'); +} + +TEST(Pcap, ReplaysMessagesFromClassicCapture) { + std::vector decoded; + itch::transport::PcapReader reader {[&](const Message& msg) { decoded.push_back(msg); }}; + + const auto mold = itch::test::moldudp64_packet( + "SESSION01", + 1, + {itch::test::system_event_payload(1, 'O'), itch::test::system_event_payload(2, 'C')} + ); + const auto frame = itch::test::ethernet_ipv4_udp_frame(26477, mold); + const auto file = itch::test::classic_pcap({frame}); + + EXPECT_TRUE(reader.read(std::span {file})); + EXPECT_EQ(reader.udp_datagrams(), 1U); + EXPECT_EQ(reader.messages_decoded(), 2U); + EXPECT_EQ(event_codes(decoded), "OC"); +} + +TEST(Pcap, PortFilterDropsNonMatchingDatagrams) { + std::vector decoded; + itch::transport::PcapReader reader {[&](const Message& msg) { decoded.push_back(msg); }}; + reader.set_udp_port_filter(9999); + + const auto mold = + itch::test::moldudp64_packet("S", 1, {itch::test::system_event_payload(1, 'O')}); + const auto frame = itch::test::ethernet_ipv4_udp_frame(26477, mold); + const auto file = itch::test::classic_pcap({frame}); + + EXPECT_TRUE(reader.read(std::span {file})); + EXPECT_EQ(reader.messages_decoded(), 0U); +} + +TEST(Pcap, UnrecognizedBufferIsRejected) { + itch::transport::PcapReader reader {[](const Message&) {}}; + std::vector junk(64, std::byte {0x11}); + EXPECT_FALSE(reader.read(std::span {junk})); +} + +TEST(SequenceTracker, IgnoresDuplicateReplayedMessages) { + itch::transport::SequenceTracker tracker; + EXPECT_EQ(tracker.observe("S", 1, 3), 0U); // messages 1,2,3 + EXPECT_EQ(tracker.observe("S", 2, 3), 0U); // overlapping replay 2,3,4 + EXPECT_EQ(tracker.messages_seen(), 4U); // only 1,2,3,4 counted once + EXPECT_EQ(tracker.expected_next("S"), 5U); +} From 8886eeef6604b8239f2a9e1047a06838957c6837 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Mon, 29 Jun 2026 00:31:59 +0100 Subject: [PATCH 027/144] Add Phase 2 production order-book engine Implement the FEATURES.md Phase 2 full-market book engine alongside the existing single-symbol LimitOrderBook (which is retained unchanged): - L3Book: allocation-light order-level book. Orders live in a reusable object pool linked into intrusive FIFO queues; price levels are flat sorted ladders; order lookup uses a flat open-addressed map (OrderIndex), removing the per-order shared_ptr / list node / map node of the original design. - BookManager: maintains a book per symbol from one pass over the feed, routed by stock locate code in O(1), with an optional symbol-universe filter. - BBO change events, L2 aggregated and L3 order-level depth snapshots, and trade-tape extraction (tape.hpp) honouring the printable flag. - Zero-copy overlay parse API (overlay.hpp): lazy typed views with on-access endianness conversion; ~1.8x faster than eager when touching a few fields. - Shared wire-layout metadata (detail/wire.hpp) now backs both the parser dispatch table and the overlay. Adds book/overlay tests, a book benchmark with published numbers, a book-engine example, and README/CHANGELOG entries. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build.yml | 10 +- .gitignore | 3 + CHANGELOG.md | 22 +++ README.md | 65 +++++++ VERSION.txt | 2 +- benchmarks/CMakeLists.txt | 10 +- benchmarks/book_bench.cpp | 127 ++++++++++++ examples/CMakeLists.txt | 8 + examples/book/book_engine_example.cpp | 48 +++++ include/itch/book/book_manager.hpp | 84 ++++++++ include/itch/book/l3_book.hpp | 166 ++++++++++++++++ include/itch/book/order_index.hpp | 144 ++++++++++++++ include/itch/detail/wire.hpp | 69 +++++++ include/itch/overlay.hpp | 262 +++++++++++++++++++++++++ include/itch/tape.hpp | 34 ++++ src/CMakeLists.txt | 2 + src/book/book_manager.cpp | 223 +++++++++++++++++++++ src/book/l3_book.cpp | 268 ++++++++++++++++++++++++++ src/parser.cpp | 46 +---- tests/CMakeLists.txt | 2 + tests/book/test_book.cpp | 174 +++++++++++++++++ tests/book/test_overlay.cpp | 83 ++++++++ tests/transport/frame_builders.hpp | 24 +++ 23 files changed, 1830 insertions(+), 46 deletions(-) create mode 100644 benchmarks/book_bench.cpp create mode 100644 examples/book/book_engine_example.cpp create mode 100644 include/itch/book/book_manager.hpp create mode 100644 include/itch/book/l3_book.hpp create mode 100644 include/itch/book/order_index.hpp create mode 100644 include/itch/detail/wire.hpp create mode 100644 include/itch/overlay.hpp create mode 100644 include/itch/tape.hpp create mode 100644 src/book/book_manager.cpp create mode 100644 src/book/l3_book.cpp create mode 100644 tests/book/test_book.cpp create mode 100644 tests/book/test_overlay.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 93e64e0..4e0e87b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -294,8 +294,8 @@ jobs: -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ -DVCPKG_TARGET_TRIPLET=x64-linux - - name: Build benchmark - run: cmake --build $BUILD_DIR --target parser_bench + - name: Build benchmarks + run: cmake --build $BUILD_DIR --target parser_bench book_bench - name: Generate a small synthetic ITCH sample run: | @@ -307,5 +307,7 @@ jobs: out.write(frame * 1_000_000) PY - - name: Run benchmark (smoke) - run: $BUILD_DIR/benchmarks/parser_bench sample.itch --benchmark_min_time=1x + - name: Run benchmarks (smoke) + run: | + $BUILD_DIR/benchmarks/parser_bench sample.itch --benchmark_min_time=1x + $BUILD_DIR/benchmarks/book_bench sample.itch --benchmark_min_time=1x diff --git a/.gitignore b/.gitignore index b07f192..5bf5345 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,10 @@ *.gz *.pdf build/ +build-*/ install/ +scratch_* +*.itch *-workspace *_ITCH50 scripts.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index c0a44be..7e24bf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Phase 2 — Production order-book engine** ([FEATURES.md](FEATURES.md)). A + full-market book engine alongside the existing single-symbol `LimitOrderBook` + (which is unchanged and retained): + - `itch::book::L3Book` (`itch/book/l3_book.hpp`): an allocation-light, order-level + book. Orders live in a reusable object pool linked into intrusive FIFO queues, + price levels are flat sorted ladders, and order lookup by reference number uses + a flat open-addressed map (`itch/book/order_index.hpp`), removing the per-order + `std::shared_ptr`, list node, and map node of the original design. + - `itch::book::BookManager` (`itch/book/book_manager.hpp`): maintains a book per + symbol from one pass over the feed, routed by stock locate code in O(1), with an + optional symbol universe filter. + - Best-bid/offer change events (`set_bbo_callback`) and configurable L2 aggregated + and L3 order-level depth snapshots (`L3Book::depth`, `L3Book::orders_at`). + - Trade-tape extraction (`itch/tape.hpp`, `set_trade_callback`) from `E`/`C`/`P`/`Q` + messages, exposing the `printable` flag to distinguish displayable from hidden + prints. + - A zero-copy overlay parse API (`itch/overlay.hpp`): lazy typed views over the + raw buffer with on-access endianness conversion, for callers that touch only a + few fields per message. + - A book/overlay benchmark (`benchmarks/book_bench.cpp`) and a book-engine example + (`examples/book/book_engine_example.cpp`), with measured numbers in the README. + - **Phase 1 — Real feed ingestion** ([FEATURES.md](FEATURES.md)). The library can now consume ITCH as it is actually delivered, not only pre-stripped message streams: diff --git a/README.md b/README.md index 664ecaa..156d2bd 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,13 @@ The design of this ITCH parser is guided by three principles: multicast framing, SoupBinTCP (Glimpse/recovery) framing, and `.pcap`/`.pcapng` captures — with per-session sequence tracking and gap detection. No libpcap dependency. See [Feed Ingestion](#feed-ingestion-transport). +- **Full-Market Book Engine**: Reconstruct every symbol on the feed in one pass + with an allocation-light L3 order book (object pool, intrusive FIFO levels, flat + ladders, open-addressed O(1) order lookup), with BBO change events, L2/L3 depth + snapshots, and trade-tape extraction. See [Book Engine](#full-market-book-engine). +- **Zero-Copy Overlay API**: Inspect raw frames through lazy typed views that + convert only the fields you read, for hot paths that touch a few fields per + message. - **High Throughput**: Multi-gigabyte-per-second parsing on modern hardware (see [Benchmarks](#benchmarks) for measured numbers). - **Allocation-Free Core**: The callback-based parsing loop performs zero dynamic memory allocations, minimizing latency and jitter. - **Type-Safe API**: All ITCH messages are deserialized into a `std::variant` of dedicated, packed `struct`s, ensuring compile-time safety. @@ -481,6 +488,46 @@ For a live or captured MoldUDP64 datagram you already have in memory, call `MoldUdp64Decoder::decode_packet(span)` directly; for a SoupBinTCP byte stream, push segments through `SoupBinDecoder::feed(span)` as they arrive. +### Full-Market Book Engine + +The original `LimitOrderBook` reconstructs a single, pre-selected symbol. The +Phase 2 `itch::book::BookManager` reconstructs **every** symbol on the feed in one +pass, routing each message to that security's book by stock locate code in O(1). +Each book (`itch::book::L3Book`) is allocation-light: orders live in a reusable +object pool linked into intrusive FIFO queues, price levels are kept in flat +sorted ladders, and order lookup by reference number uses a flat open-addressed +map, so there is no per-order heap allocation or atomic refcount on the hot path. + +```cpp +#include "itch/book/book_manager.hpp" +#include "itch/parser.hpp" + +itch::book::BookManager manager; +// manager.track_symbol("AAPL"); // optional: restrict to a universe + +manager.set_bbo_callback([](const itch::book::L3Book& book, const itch::book::Bbo& bbo) { + // best bid/offer for book.symbol() just changed +}); +manager.set_trade_callback([](const itch::Trade& trade) { + // trade.printable distinguishes displayable prints from hidden ones +}); + +itch::Parser parser; +parser.parse(buffer.data(), buffer.size(), [&](const itch::Message& msg) { + manager.process(msg); +}); + +const itch::book::L3Book* book = manager.book_for_symbol("AAPL"); +auto top = book->bbo(); // best bid/offer +auto l2 = book->depth(itch::book::Side::buy, 5); // top-5 aggregated bids +auto l3 = book->orders_at(itch::book::Side::buy, top.bid_price.raw()); // order-level +``` + +For callers that touch only a few fields per message, `itch/overlay.hpp` provides +a zero-copy alternative to the eager parser: `for_each_message(buffer, cb)` yields +a `MessageView` (and typed views like `AddOrderView`) that decode each field +lazily on access. + --- ## API Reference: ITCH 5.0 Message Types @@ -557,6 +604,24 @@ The numbers below were produced by [`benchmarks/parser_bench.cpp`](benchmarks/pa The collect/filter paths are slower than the callback path because they allocate and copy each parsed message into a `std::vector`; the callback path does neither, which is why it is the recommended interface for latency-sensitive processing. +#### Book engine and overlay + +These are produced by [`benchmarks/book_bench.cpp`](benchmarks/book_bench.cpp). +The throughput figures below were measured on a synthetic, churn-heavy stream +(adds and deletes around a moving mid across 100 symbols) on the same hardware, +Clang 22 `-DCMAKE_BUILD_TYPE=Release`, C++23; book-rebuild throughput on real +data depends heavily on the depth and churn profile of the feed. + +| Configuration | Throughput | +| ------------- | ---------- | +| `BookManager::process` — full multi-symbol L3 reconstruction | ~150 MiB/s | +| Eager `parse`, touching one field per message | ~5.4 GiB/s | +| Zero-copy `overlay::for_each_message`, touching one field per message | **~9.7 GiB/s** | + +The overlay is materially faster than the eager decoder when only a few fields are +read, because it never decodes the fields the caller does not touch. Reproduce +with `./build/benchmarks/book_bench `. + ### How to reproduce ```bash diff --git a/VERSION.txt b/VERSION.txt index 26aaba0..f0bb29e 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.2.0 +1.3.0 diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 061d816..778c432 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -15,7 +15,15 @@ endif() add_executable(parser_bench parser_bench.cpp) target_link_libraries( - parser_bench PRIVATE + parser_bench PRIVATE + itch::itch + benchmark::benchmark + benchmark::benchmark_main +) + +add_executable(book_bench book_bench.cpp) +target_link_libraries( + book_bench PRIVATE itch::itch benchmark::benchmark benchmark::benchmark_main diff --git a/benchmarks/book_bench.cpp b/benchmarks/book_bench.cpp new file mode 100644 index 0000000..02b19b9 --- /dev/null +++ b/benchmarks/book_bench.cpp @@ -0,0 +1,127 @@ +/** + * @file book_bench.cpp + * @brief Benchmarks for the Phase 2 book engine and the zero-copy overlay. + * + * Measures three things against an in-memory ITCH buffer (file I/O excluded): + * - BM_BookRebuild: full multi-symbol book reconstruction through BookManager. + * - BM_EagerTouch: eager parse touching one field per message (the baseline). + * - BM_OverlayTouch: lazy overlay framing touching the same one field, which + * should be cheaper because the other fields are never decoded. + * + * Usage: + * ./book_bench [google benchmark options] + */ + +#include + +#include +#include +#include +#include +#include + +#include "itch/book/book_manager.hpp" +#include "itch/overlay.hpp" +#include "itch/parser.hpp" + +namespace data { +// NOLINTNEXTLINE +std::string g_data_filename {}; +} // namespace data + +namespace { + +auto load_file(const std::string& path) -> std::vector { + std::ifstream file {path, std::ios::binary}; + file.seekg(0, std::ios::end); + const auto size = file.tellg(); + file.seekg(0, std::ios::beg); + std::vector buffer(static_cast(size)); + file.read(reinterpret_cast(buffer.data()), size); // NOLINT + return buffer; +} + +class BookBenchmark : public benchmark::Fixture { + public: + std::vector itch_data; + + void SetUp(::benchmark::State& state) override { + if (data::g_data_filename.empty()) { + state.SkipWithError("ITCH data file not provided."); + return; + } + itch_data = load_file(data::g_data_filename); + if (itch_data.empty()) { + state.SkipWithError("Failed to read ITCH data file."); + } + } + void TearDown(const ::benchmark::State&) override { + itch_data.clear(); + itch_data.shrink_to_fit(); + } +}; + +} // namespace + +BENCHMARK_F(BookBenchmark, BM_BookRebuild)(benchmark::State& state) { + std::size_t total_bytes = 0; + for ([[maybe_unused]] auto iter : state) { + itch::Parser parser; + itch::book::BookManager manager; + parser.parse(std::span {itch_data}, [&](const itch::Message& msg) { + manager.process(msg); + }); + benchmark::DoNotOptimize(manager.book_count()); + total_bytes += itch_data.size(); + } + state.SetBytesProcessed(static_cast(total_bytes)); +} + +BENCHMARK_F(BookBenchmark, BM_EagerTouch)(benchmark::State& state) { + std::size_t total_bytes = 0; + for ([[maybe_unused]] auto iter : state) { + itch::Parser parser; + std::uint64_t sink = 0; + parser.parse(std::span {itch_data}, [&](const itch::Message& msg) { + sink += std::visit( + [](const auto& inner) -> std::uint64_t { return inner.stock_locate; }, msg + ); + }); + benchmark::DoNotOptimize(sink); + total_bytes += itch_data.size(); + } + state.SetBytesProcessed(static_cast(total_bytes)); +} + +BENCHMARK_F(BookBenchmark, BM_OverlayTouch)(benchmark::State& state) { + std::size_t total_bytes = 0; + for ([[maybe_unused]] auto iter : state) { + std::uint64_t sink = 0; + itch::overlay::for_each_message( + std::span {itch_data}, + [&](const itch::overlay::MessageView& view) { sink += view.stock_locate(); } + ); + benchmark::DoNotOptimize(sink); + total_bytes += itch_data.size(); + } + state.SetBytesProcessed(static_cast(total_bytes)); +} + +auto main(int argc, char** argv) -> int { + if (argc < 2) { + std::cerr << "Usage: ./book_bench [benchmark options]\n"; + return 1; + } + data::g_data_filename = argv[1]; + for (int index = 1; index < argc - 1; ++index) { + argv[index] = argv[index + 1]; + } + --argc; + benchmark::Initialize(&argc, argv); + if (benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 6066493..60a4e57 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -23,3 +23,11 @@ target_link_libraries( itch::itch warnings::strict ) + +add_executable(book_engine_example book/book_engine_example.cpp) + +target_link_libraries( + book_engine_example PRIVATE + itch::itch + warnings::strict +) diff --git a/examples/book/book_engine_example.cpp b/examples/book/book_engine_example.cpp new file mode 100644 index 0000000..2aa4348 --- /dev/null +++ b/examples/book/book_engine_example.cpp @@ -0,0 +1,48 @@ +#include +#include +#include +#include + +#include "itch/book/book_manager.hpp" +#include "itch/parser.hpp" + +// Reconstructs every symbol's order book in one pass over an ITCH file using the +// multi-symbol BookManager, printing best-bid/offer changes and the trade tape. +auto main(int argc, char* argv[]) -> int { + if (argc < 2) { + std::println(stderr, "Usage: {} [symbol]", argv[0]); + return 1; + } + + std::ifstream file {argv[1], std::ios::binary}; + if (!file) { + std::println(stderr, "Error: cannot open '{}'.", argv[1]); + return 1; + } + std::vector buffer {std::istreambuf_iterator {file}, {}}; + + itch::book::BookManager manager; + if (argc >= 3) { + manager.track_symbol(argv[2]); // Restrict to one symbol when requested. + } + + std::uint64_t bbo_updates = 0; + std::uint64_t trades = 0; + manager.set_bbo_callback([&](const itch::book::L3Book&, const itch::book::Bbo&) { + ++bbo_updates; + }); + manager.set_trade_callback([&](const itch::Trade&) { ++trades; }); + + itch::Parser parser; + parser.parse(buffer.data(), buffer.size(), [&](const itch::Message& msg) { + manager.process(msg); + }); + + std::println( + "Reconstructed {} books, {} BBO changes, {} trades.", + manager.book_count(), + bbo_updates, + trades + ); + return 0; +} diff --git a/include/itch/book/book_manager.hpp b/include/itch/book/book_manager.hpp new file mode 100644 index 0000000..329ae6c --- /dev/null +++ b/include/itch/book/book_manager.hpp @@ -0,0 +1,84 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "itch/book/l3_book.hpp" +#include "itch/messages.hpp" +#include "itch/tape.hpp" + +namespace itch::book { + +/// @brief Maintains a full-market set of order books from a single pass over the +/// feed. +/// +/// Every order, execution, cancel, delete, and replace message carries a stock +/// locate code; the manager routes each message to that security's `L3Book` in +/// O(1) through a flat, locate-indexed table, reconstructing every symbol on the +/// feed at once rather than a single pre-selected one. It optionally restricts +/// work to a chosen universe of symbols, emits best-bid/offer change events as +/// they happen, and extracts the trade tape. +class BookManager { + public: + /// @brief Invoked when a book's best bid or offer changes. + using BboCallback = std::function; + + BookManager() = default; + + /// @brief Processes one parsed ITCH message, updating the relevant book and + /// emitting BBO/trade events as appropriate. + auto process(const Message& message) -> void; + + /// @brief Installs the best-bid/offer change callback (empty clears it). + auto set_bbo_callback(BboCallback callback) -> void { m_bbo_callback = std::move(callback); } + + /// @brief Installs the trade-tape callback (empty clears it). + auto set_trade_callback(TradeCallback callback) -> void { + m_trade_callback = std::move(callback); + } + + /// @brief Restricts tracking to the given symbol (call once per symbol). When + /// no symbol is added, every symbol on the feed is tracked. + auto track_symbol(std::string symbol) -> void { m_universe.insert(std::move(symbol)); } + + /// @brief The book for a locate code, or nullptr if none is tracked. + [[nodiscard]] auto book(std::uint16_t stock_locate) const -> const L3Book*; + + /// @brief The book for a symbol, or nullptr if none is tracked. + [[nodiscard]] auto book_for_symbol(std::string_view symbol) const -> const L3Book*; + + /// @brief The number of books currently maintained. + [[nodiscard]] auto book_count() const noexcept -> std::size_t { return m_book_count; } + + /// @brief The symbol associated with a locate code (empty if unknown). + [[nodiscard]] auto symbol_for_locate(std::uint16_t stock_locate) const -> std::string_view; + + private: + struct BookEntry { + L3Book book; + Bbo last_bbo {}; + }; + + // Returns the entry for a locate, creating it if the symbol is in-universe. + auto ensure_entry(std::uint16_t stock_locate, std::string_view symbol) -> BookEntry*; + // Returns the existing entry for a locate, or nullptr. + [[nodiscard]] auto entry(std::uint16_t stock_locate) const -> BookEntry*; + // Emits a BBO event if the book's top has changed since last seen. + auto emit_bbo_if_changed(BookEntry& target) -> void; + // Whether the symbol should be tracked given the configured universe. + [[nodiscard]] auto in_universe(std::string_view symbol) const -> bool; + + std::vector> m_books_by_locate; ///< Indexed by locate. + std::vector m_symbol_by_locate; ///< Locate -> symbol. + std::unordered_set m_universe; ///< Empty == track all. + BboCallback m_bbo_callback {}; + TradeCallback m_trade_callback {}; + std::size_t m_book_count {0}; +}; + +} // namespace itch::book diff --git a/include/itch/book/l3_book.hpp b/include/itch/book/l3_book.hpp new file mode 100644 index 0000000..2235db2 --- /dev/null +++ b/include/itch/book/l3_book.hpp @@ -0,0 +1,166 @@ +#pragma once + +#include +#include +#include +#include + +#include "itch/book/order_index.hpp" +#include "itch/price.hpp" + +namespace itch::book { + +/// @brief Which side of the book an order rests on. +enum class Side : char { + buy = 'B', ///< A bid. + sell = 'S', ///< An offer/ask. +}; + +/// @brief Aggregated state of one price level, for L2 depth snapshots. +struct DepthLevel { + StandardPrice price {}; ///< The level's limit price. + std::uint64_t shares {0}; ///< Total displayed shares resting at the level. + std::uint32_t order_count {0}; ///< Number of resting orders at the level. +}; + +/// @brief A single resting order, for L3 order-level snapshots. +struct OrderView { + std::uint64_t reference_number {0}; ///< Exchange order reference number. + std::uint32_t shares {0}; ///< Shares still resting. + StandardPrice price {}; ///< Limit price. +}; + +/// @brief Best bid and offer of a single book. +struct Bbo { + bool has_bid {false}; ///< Whether a bid side exists. + bool has_ask {false}; ///< Whether an ask side exists. + StandardPrice bid_price {}; ///< Best (highest) bid price. + std::uint64_t bid_shares {0}; ///< Shares at the best bid. + StandardPrice ask_price {}; ///< Best (lowest) ask price. + std::uint64_t ask_shares {0}; ///< Shares at the best ask. + + [[nodiscard]] friend auto operator==(const Bbo&, const Bbo&) noexcept -> bool = default; +}; + +/// @brief A single-symbol, order-level (L3) limit order book with allocation-light +/// internals. +/// +/// Unlike the original `LimitOrderBook`, which stored each order in a +/// `std::shared_ptr` inside a `std::list` and each price level in a `std::map`, +/// this engine holds orders in a reusable object pool (a flat vector with a free +/// list) linked into intrusive FIFO queues, and keeps each side's price levels in +/// a flat, sorted vector. There is no per-order heap allocation, no atomic +/// refcount, and no per-level node allocation on the hot path; order lookup by +/// reference number is O(1) and the best bid/offer is the front of a ladder. +class L3Book { + public: + /// @brief Constructs a book, optionally tagged with its stock symbol. + explicit L3Book(std::string symbol = {}); + + /// @brief Sets the stock symbol associated with this book. + auto set_symbol(std::string symbol) -> void { m_symbol = std::move(symbol); } + + /// @brief The stock symbol associated with this book (may be empty). + [[nodiscard]] auto symbol() const noexcept -> const std::string& { return m_symbol; } + + /// @brief Adds a new resting order to the book (ITCH `A`/`F`). + auto add_order( + std::uint64_t reference_number, Side side, std::uint32_t shares, std::uint32_t price + ) -> void; + + /// @brief Removes `shares` from an order on execution (ITCH `E`/`C`), + /// deleting it when fully filled. Returns the shares actually removed. + auto execute_order(std::uint64_t reference_number, std::uint32_t shares) -> std::uint32_t; + + /// @brief Removes `shares` from an order on a partial cancel (ITCH `X`), + /// deleting it when it reaches zero. Returns the shares actually removed. + auto reduce_order(std::uint64_t reference_number, std::uint32_t shares) -> std::uint32_t; + + /// @brief Deletes an order in its entirety (ITCH `D`). + auto delete_order(std::uint64_t reference_number) -> void; + + /// @brief Replaces an order with a new reference number, size, and price + /// on the same side (ITCH `U`). + auto replace_order( + std::uint64_t old_reference_number, + std::uint64_t new_reference_number, + std::uint32_t shares, + std::uint32_t price + ) -> void; + + /// @brief Whether an order with the given reference number is resting. + [[nodiscard]] auto contains(std::uint64_t reference_number) const -> bool; + + /// @brief The raw limit price of a resting order, if present. + [[nodiscard]] auto order_price(std::uint64_t reference_number) const + -> std::optional; + + /// @brief The side of a resting order, if present. + [[nodiscard]] auto order_side(std::uint64_t reference_number) const -> std::optional; + + /// @brief The current best bid and offer. + [[nodiscard]] auto bbo() const -> Bbo; + + /// @brief Aggregated L2 depth for a side, best level first. + /// + /// @param side The side to report. + /// @param max_levels The maximum number of levels to return (0 == all). + [[nodiscard]] auto depth(Side side, std::size_t max_levels = 0) const + -> std::vector; + + /// @brief The resting orders at a given price on a side, in time priority. + [[nodiscard]] auto orders_at(Side side, std::uint32_t price) const -> std::vector; + + /// @brief The number of active price levels on a side. + [[nodiscard]] auto level_count(Side side) const noexcept -> std::size_t; + + /// @brief Whether the book has no resting orders on either side. + [[nodiscard]] auto empty() const noexcept -> bool { return m_index.empty(); } + + private: + /// @brief Sentinel index meaning "no node". + static constexpr std::uint32_t NIL = 0xFFFFFFFFU; + + /// @brief One resting order in the object pool, linked into a level's FIFO. + struct OrderNode { + std::uint64_t reference_number {0}; + std::uint32_t shares {0}; + std::uint32_t price {0}; + Side side {Side::buy}; + std::uint32_t next {NIL}; ///< Next order in level FIFO, or next free node. + std::uint32_t prev {NIL}; ///< Previous order in level FIFO. + }; + + /// @brief One price level holding the head/tail of an intrusive FIFO queue. + struct Level { + std::uint32_t price {0}; + std::uint64_t total_shares {0}; + std::uint32_t order_count {0}; + std::uint32_t head {NIL}; + std::uint32_t tail {NIL}; + }; + + [[nodiscard]] auto side_levels(Side side) noexcept -> std::vector&; + [[nodiscard]] auto side_levels(Side side) const noexcept -> const std::vector&; + + // Allocates a node from the free list (or grows the pool) and returns its index. + auto allocate_node() -> std::uint32_t; + // Returns a node to the free list. + auto free_node(std::uint32_t node_index) -> void; + // Finds the index of the level at `price` on `side`, or NIL if absent. + [[nodiscard]] auto find_level(Side side, std::uint32_t price) const -> std::uint32_t; + // Finds, or inserts in sorted order, the level at `price`; returns its index. + auto find_or_create_level(Side side, std::uint32_t price) -> std::uint32_t; + // Unlinks a node from its level FIFO and removes the level if it empties. + auto unlink_node(std::uint32_t node_index) -> void; + + std::string m_symbol; + std::vector m_pool; + std::uint32_t m_free_head {NIL}; + std::vector m_bids; ///< Sorted high to low; front is the best bid. + std::vector m_asks; ///< Sorted low to high; front is the best ask. + // Reference-number -> pool index for O(1), allocation-free order lookup. + OrderIndex m_index; +}; + +} // namespace itch::book diff --git a/include/itch/book/order_index.hpp b/include/itch/book/order_index.hpp new file mode 100644 index 0000000..19b49a3 --- /dev/null +++ b/include/itch/book/order_index.hpp @@ -0,0 +1,144 @@ +#pragma once + +#include +#include +#include + +namespace itch::book { + +/// @brief A flat, open-addressed hash map from order reference number to pool +/// index, used for O(1) order lookup without per-order heap allocation. +/// +/// `std::unordered_map` allocates a node on every insert and frees one on every +/// erase, which on a feed of hundreds of millions of add/delete messages dominates +/// the book's cost and defeats the allocation-free goal. This map stores its slots +/// in a single contiguous vector, probes linearly (cache friendly), and uses +/// backward-shift deletion so heavy add/cancel churn does not accumulate +/// tombstones. It only allocates when it grows. +class OrderIndex { + public: + /// @brief Sentinel returned by `find` when a key is absent. + static constexpr std::uint32_t NPOS = 0xFFFFFFFFU; + + OrderIndex() { rehash(INITIAL_CAPACITY); } + + /// @brief The number of stored keys. + [[nodiscard]] auto size() const noexcept -> std::size_t { return m_count; } + + /// @brief Whether the map holds no keys. + [[nodiscard]] auto empty() const noexcept -> bool { return m_count == 0; } + + /// @brief Returns the value for `key`, or `NPOS` if absent. + [[nodiscard]] auto find(std::uint64_t key) const noexcept -> std::uint32_t { + std::size_t slot = hash(key) & m_mask; + while (m_slots[slot].used) { + if (m_slots[slot].key == key) { + return m_slots[slot].value; + } + slot = (slot + 1) & m_mask; + } + return NPOS; + } + + /// @brief Whether `key` is present. + [[nodiscard]] auto contains(std::uint64_t key) const noexcept -> bool { + return find(key) != NPOS; + } + + /// @brief Inserts or overwrites the value for `key`. + auto insert(std::uint64_t key, std::uint32_t value) -> void { + if ((m_count + 1) * LOAD_FACTOR_DEN >= m_slots.size() * LOAD_FACTOR_NUM) { + rehash(m_slots.size() * 2); + } + std::size_t slot = hash(key) & m_mask; + while (m_slots[slot].used) { + if (m_slots[slot].key == key) { + m_slots[slot].value = value; + return; + } + slot = (slot + 1) & m_mask; + } + m_slots[slot] = Slot {key, value, true}; + ++m_count; + } + + /// @brief Removes `key` if present, repairing the probe chain in place. + auto erase(std::uint64_t key) -> void { + std::size_t slot = hash(key) & m_mask; + while (m_slots[slot].used) { + if (m_slots[slot].key == key) { + remove_at(slot); + return; + } + slot = (slot + 1) & m_mask; + } + } + + /// @brief Drops all keys (retaining capacity). + auto clear() -> void { + for (auto& slot : m_slots) { + slot.used = false; + } + m_count = 0; + } + + private: + struct Slot { + std::uint64_t key {0}; + std::uint32_t value {0}; + bool used {false}; + }; + + static constexpr std::size_t INITIAL_CAPACITY = 1024; + static constexpr std::size_t LOAD_FACTOR_NUM = 7; // Grow past 70% load. + static constexpr std::size_t LOAD_FACTOR_DEN = 10; + + // A finalizing mix (splitmix64) so dense, sequential reference numbers spread + // across the table instead of clustering. + [[nodiscard]] static auto hash(std::uint64_t key) noexcept -> std::size_t { + key ^= key >> 33; + key *= 0xFF51AFD7ED558CCDULL; + key ^= key >> 33; + key *= 0xC4CEB9FE1A85EC53ULL; + key ^= key >> 33; + return static_cast(key); + } + + auto remove_at(std::size_t hole) -> void { + m_slots[hole].used = false; + --m_count; + std::size_t next = (hole + 1) & m_mask; + while (m_slots[next].used) { + const std::size_t home = hash(m_slots[next].key) & m_mask; + // Move the entry into the hole if its home position is not within the + // cyclic interval (hole, next], i.e. the hole sits between its home + // and its current slot. + const bool movable = + (hole <= next) ? (home <= hole || home > next) : (home <= hole && home > next); + if (movable) { + m_slots[hole] = m_slots[next]; + m_slots[next].used = false; + hole = next; + } + next = (next + 1) & m_mask; + } + } + + auto rehash(std::size_t new_capacity) -> void { + std::vector old = std::move(m_slots); + m_slots.assign(new_capacity, Slot {}); + m_mask = new_capacity - 1; + m_count = 0; + for (const auto& slot : old) { + if (slot.used) { + insert(slot.key, slot.value); + } + } + } + + std::vector m_slots; + std::size_t m_count {0}; + std::size_t m_mask {0}; +}; + +} // namespace itch::book diff --git a/include/itch/detail/wire.hpp b/include/itch/detail/wire.hpp new file mode 100644 index 0000000..93589c1 --- /dev/null +++ b/include/itch/detail/wire.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include +#include + +#include "itch/messages.hpp" + +/// @file +/// @brief Single source of truth for the ITCH 5.0 wire layout metadata shared by +/// the eager parser, the zero-copy overlay, and the encoder. +/// +/// Keeping the per-type wire size and the canonical list of message types in one +/// place means the dispatch table, the overlay accessors, and the writer can +/// never drift apart from one another. + +namespace itch::detail { + +/// @brief The on-wire ITCH timestamp is 48 bits (6 bytes), but every message +/// struct stores it in a 64-bit field, so each struct is exactly two +/// bytes wider than its encoding. The timestamp is the only field whose +/// storage width differs from its wire width. +constexpr std::size_t TIMESTAMP_STRUCT_PADDING = sizeof(std::uint64_t) - 6; + +/// @brief The exact on-wire size, in bytes, of a fully formed message of type T. +template +constexpr std::size_t WIRE_SIZE = sizeof(MsgType) - TIMESTAMP_STRUCT_PADDING; + +// Lock the padding assumption to the spec lengths so a future struct change that +// breaks the derivation is caught at compile time rather than at runtime. +static_assert(WIRE_SIZE == 12); +static_assert(WIRE_SIZE == 39); +static_assert(WIRE_SIZE == 36); +static_assert(WIRE_SIZE == 50); +static_assert(WIRE_SIZE == 48); + +/// @brief Invokes `visitor.template operator()(type_byte)` once for each +/// ITCH 5.0 message type, in a fixed order. +/// +/// This is the canonical registry of message types. Both the parser's dispatch +/// table and any other component that needs to enumerate the message set build +/// on it, so the set is defined exactly once. +template +constexpr auto for_each_message_type(Visitor&& visitor) -> void { + visitor.template operator()('S'); + visitor.template operator()('R'); + visitor.template operator()('H'); + visitor.template operator()('Y'); + visitor.template operator()('L'); + visitor.template operator()('V'); + visitor.template operator()('W'); + visitor.template operator()('K'); + visitor.template operator()('J'); + visitor.template operator()('h'); + visitor.template operator()('A'); + visitor.template operator()('F'); + visitor.template operator()('E'); + visitor.template operator()('C'); + visitor.template operator()('X'); + visitor.template operator()('D'); + visitor.template operator()('U'); + visitor.template operator()('P'); + visitor.template operator()('Q'); + visitor.template operator()('B'); + visitor.template operator()('I'); + visitor.template operator()('N'); + visitor.template operator()('O'); +} + +} // namespace itch::detail diff --git a/include/itch/overlay.hpp b/include/itch/overlay.hpp new file mode 100644 index 0000000..8ee7b1d --- /dev/null +++ b/include/itch/overlay.hpp @@ -0,0 +1,262 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "itch/detail/wire.hpp" +#include "itch/parser.hpp" + +namespace itch::overlay { + +namespace detail { + +/// @brief Reads an integral field of width sizeof(T) at `offset`, converting from +/// network (big-endian) order on access. +template +[[nodiscard]] inline auto read_field(const std::byte* base, std::size_t offset) noexcept + -> FieldType { + FieldType value {}; + std::memcpy(&value, base + offset, sizeof(FieldType)); + if constexpr (std::is_integral_v && sizeof(FieldType) > 1) { + return utils::from_big_endian(value); + } else { + return value; + } +} + +/// @brief Reads the 48-bit ITCH timestamp at `offset` into a 64-bit value. +[[nodiscard]] inline auto read_timestamp(const std::byte* base, std::size_t offset) noexcept + -> std::uint64_t { + const auto high = read_field(base, offset); + const auto low = read_field(base, offset + 2); + constexpr int LOWER_SHIFT = 32; + return (static_cast(high) << LOWER_SHIFT) | low; +} + +/// @brief Views an 8-byte fixed-width stock field as a string_view (untrimmed). +[[nodiscard]] inline auto read_stock(const std::byte* base, std::size_t offset) noexcept + -> std::string_view { + const void* raw = base + offset; + return std::string_view {static_cast(raw), 8}; +} + +// Common field offsets shared by every message after the 1-byte type. +constexpr std::size_t STOCK_LOCATE_OFFSET = 1; +constexpr std::size_t TRACKING_NUMBER_OFFSET = 3; +constexpr std::size_t TIMESTAMP_OFFSET = 5; + +} // namespace detail + +/// @brief A zero-copy typed view over one raw ITCH frame. +/// +/// In contrast to the eager `Parser`, which decodes every field of a message into +/// a host-order struct, an overlay view holds only a pointer to the frame in the +/// original buffer and converts each field from network byte order lazily, on the +/// access. For callers that touch only a few fields per message (for example a +/// filter that reads just the stock and price), this avoids decoding the fields +/// they never look at. The view is non-owning: it is valid only while the +/// underlying buffer lives. +class MessageView { + public: + constexpr MessageView() noexcept = default; + explicit MessageView(const std::byte* data, std::size_t size) noexcept + : m_data {data}, m_size {size} {} + + /// @brief The one-byte message type. + [[nodiscard]] auto type() const noexcept -> char { + return static_cast(static_cast(m_data[0])); + } + + /// @brief The locate code identifying the security. + [[nodiscard]] auto stock_locate() const noexcept -> std::uint16_t { + return detail::read_field(m_data, detail::STOCK_LOCATE_OFFSET); + } + + /// @brief The Nasdaq internal tracking number. + [[nodiscard]] auto tracking_number() const noexcept -> std::uint16_t { + return detail::read_field(m_data, detail::TRACKING_NUMBER_OFFSET); + } + + /// @brief The message timestamp (nanoseconds past midnight). + [[nodiscard]] auto timestamp() const noexcept -> std::uint64_t { + return detail::read_timestamp(m_data, detail::TIMESTAMP_OFFSET); + } + + /// @brief The raw frame size in bytes. + [[nodiscard]] auto size() const noexcept -> std::size_t { return m_size; } + + /// @brief Pointer to the raw frame (the type byte). + [[nodiscard]] auto data() const noexcept -> const std::byte* { return m_data; } + + /// @brief Reads an arbitrary integral field at a byte offset, big-endian. + template + [[nodiscard]] auto read(std::size_t offset) const noexcept -> FieldType { + return detail::read_field(m_data, offset); + } + + protected: + const std::byte* m_data {nullptr}; + std::size_t m_size {0}; +}; + +/// @brief Lazy view of an Add Order (`A`) message. +class AddOrderView : public MessageView { + public: + using MessageView::MessageView; + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + [[nodiscard]] auto buy_sell_indicator() const noexcept -> char { return read(19); } + [[nodiscard]] auto shares() const noexcept -> std::uint32_t { return read(20); } + [[nodiscard]] auto stock() const noexcept -> std::string_view { + return detail::read_stock(m_data, 24); + } + [[nodiscard]] auto price() const noexcept -> std::uint32_t { return read(32); } +}; + +/// @brief Lazy view of an Order Executed (`E`) message. +class OrderExecutedView : public MessageView { + public: + using MessageView::MessageView; + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + [[nodiscard]] auto executed_shares() const noexcept -> std::uint32_t { + return read(19); + } + [[nodiscard]] auto match_number() const noexcept -> std::uint64_t { + return read(23); + } +}; + +/// @brief Lazy view of an Order Executed With Price (`C`) message. +class OrderExecutedWithPriceView : public MessageView { + public: + using MessageView::MessageView; + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + [[nodiscard]] auto executed_shares() const noexcept -> std::uint32_t { + return read(19); + } + [[nodiscard]] auto match_number() const noexcept -> std::uint64_t { + return read(23); + } + [[nodiscard]] auto printable() const noexcept -> char { return read(31); } + [[nodiscard]] auto execution_price() const noexcept -> std::uint32_t { + return read(32); + } +}; + +/// @brief Lazy view of an Order Cancel (`X`) message. +class OrderCancelView : public MessageView { + public: + using MessageView::MessageView; + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + [[nodiscard]] auto cancelled_shares() const noexcept -> std::uint32_t { + return read(19); + } +}; + +/// @brief Lazy view of an Order Delete (`D`) message. +class OrderDeleteView : public MessageView { + public: + using MessageView::MessageView; + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } +}; + +/// @brief Lazy view of an Order Replace (`U`) message. +class OrderReplaceView : public MessageView { + public: + using MessageView::MessageView; + [[nodiscard]] auto original_order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + [[nodiscard]] auto new_order_reference_number() const noexcept -> std::uint64_t { + return read(19); + } + [[nodiscard]] auto shares() const noexcept -> std::uint32_t { return read(27); } + [[nodiscard]] auto price() const noexcept -> std::uint32_t { return read(31); } +}; + +/// @brief Lazy view of a Trade (`P`, non-cross) message. +class NonCrossTradeView : public MessageView { + public: + using MessageView::MessageView; + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + [[nodiscard]] auto buy_sell_indicator() const noexcept -> char { return read(19); } + [[nodiscard]] auto shares() const noexcept -> std::uint32_t { return read(20); } + [[nodiscard]] auto stock() const noexcept -> std::string_view { + return detail::read_stock(m_data, 24); + } + [[nodiscard]] auto price() const noexcept -> std::uint32_t { return read(32); } + [[nodiscard]] auto match_number() const noexcept -> std::uint64_t { + return read(36); + } +}; + +/// @brief The signature for the overlay framing callback. +using ViewCallback = std::function; + +/// @brief Builds the per-type expected wire-size table at compile time, mirroring +/// the eager parser's so the overlay validates frame lengths identically. +[[nodiscard]] consteval auto build_size_table() -> std::array { + std::array table {}; + auto add = [&table](char type) { + table[static_cast(type)] = + static_cast(itch::detail::WIRE_SIZE); + }; + itch::detail::for_each_message_type(add); + return table; +} + +inline constexpr auto SIZE_TABLE = build_size_table(); + +/// @brief Frames a buffer and invokes `callback` with a zero-copy `MessageView` +/// for each well-formed message. +/// +/// Framing and length validation match `Parser`: the 2-byte length prefix is +/// honoured, unknown type bytes and undersized frames are skipped. Unlike the +/// eager parser, no fields are decoded; the callback receives a view it can +/// inspect lazily. Returns the number of views delivered. +inline auto for_each_message(std::span data, const ViewCallback& callback) + -> std::uint64_t { + std::uint64_t delivered = 0; + std::size_t offset = 0; + while (offset + sizeof(std::uint16_t) <= data.size()) { + std::uint16_t length {}; + std::memcpy(&length, data.data() + offset, sizeof(length)); + length = utils::from_big_endian(length); + offset += sizeof(std::uint16_t); + if (length == 0) { + continue; + } + if (offset + length > data.size()) { + break; + } + const std::byte* frame = data.data() + offset; + const auto message_type = static_cast(frame[0]); + offset += length; + + const std::uint16_t expected = SIZE_TABLE[message_type]; + if (expected == 0 || length < expected) { + continue; // Unknown type or undersized frame. + } + callback(MessageView {frame, length}); + ++delivered; + } + return delivered; +} + +} // namespace itch::overlay diff --git a/include/itch/tape.hpp b/include/itch/tape.hpp new file mode 100644 index 0000000..6b4378a --- /dev/null +++ b/include/itch/tape.hpp @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include + +#include "itch/price.hpp" + +namespace itch { + +/// @brief A single execution extracted from the feed (the "trade tape"). +/// +/// Trades arrive in several ITCH message types: executions of displayed orders +/// (`E`), executions at a different price (`C`), trades of non-displayable orders +/// (`P`), and cross trades (`Q`). They are normalized here into one record. The +/// `printable` flag carries the spec's displayable/non-displayable distinction so +/// consumers can correctly separate lit prints from hidden ones. +struct Trade { + std::uint64_t timestamp {0}; ///< Nanoseconds past midnight. + std::uint16_t stock_locate {0}; ///< Locate code identifying the security. + std::string symbol; ///< Stock symbol (may be empty if unknown). + StandardPrice price {}; ///< Execution price. + std::uint64_t shares {0}; ///< Executed share quantity. + std::uint64_t match_number {0}; ///< Exchange match number. + char side {'\0'}; ///< Resting order side ('B'/'S'), or '\0'. + bool printable {true}; ///< Whether the print is displayable. + bool is_cross {false}; ///< Whether this is a cross (auction) trade. + char cross_type {'\0'}; ///< Cross type for `Q` trades, else '\0'. +}; + +/// @brief Callback invoked for each extracted trade. +using TradeCallback = std::function; + +} // namespace itch diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3adcc31..712ab1c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,6 +11,8 @@ add_library( transport/moldudp64.cpp transport/soupbintcp.cpp transport/pcap.cpp + book/l3_book.cpp + book/book_manager.cpp ) add_library(itch::itch ALIAS itch) if (NOT ANDROID) diff --git a/src/book/book_manager.cpp b/src/book/book_manager.cpp new file mode 100644 index 0000000..7f0c86b --- /dev/null +++ b/src/book/book_manager.cpp @@ -0,0 +1,223 @@ +#include "itch/book/book_manager.hpp" + +#include +#include + +namespace itch::book { + +namespace { + +// Maps an ITCH buy/sell indicator byte to the book Side enum. +auto to_side(char buy_sell_indicator) noexcept -> Side { + return buy_sell_indicator == 'B' ? Side::buy : Side::sell; +} + +} // namespace + +auto BookManager::in_universe(std::string_view symbol) const -> bool { + return m_universe.empty() || m_universe.contains(std::string {symbol}); +} + +auto BookManager::entry(std::uint16_t stock_locate) const -> BookEntry* { + if (stock_locate >= m_books_by_locate.size()) { + return nullptr; + } + return m_books_by_locate[stock_locate].get(); +} + +auto BookManager::ensure_entry(std::uint16_t stock_locate, std::string_view symbol) -> BookEntry* { + if (stock_locate >= m_books_by_locate.size()) { + m_books_by_locate.resize(static_cast(stock_locate) + 1); + } + if (auto& slot = m_books_by_locate[stock_locate]) { + if (slot->book.symbol().empty() && !symbol.empty()) { + slot->book.set_symbol(std::string {symbol}); + } + return slot.get(); + } + if (!in_universe(symbol)) { + return nullptr; + } + m_books_by_locate[stock_locate] = std::make_unique(); + m_books_by_locate[stock_locate]->book.set_symbol(std::string {symbol}); + ++m_book_count; + return m_books_by_locate[stock_locate].get(); +} + +auto BookManager::emit_bbo_if_changed(BookEntry& target) -> void { + const Bbo current = target.book.bbo(); + if (current != target.last_bbo) { + target.last_bbo = current; + if (m_bbo_callback) { + m_bbo_callback(target.book, current); + } + } +} + +auto BookManager::process(const Message& message) -> void { + if (const auto* add = std::get_if(&message)) { + BookEntry* target = ensure_entry(add->stock_locate, to_string(add->stock, STOCK_LEN)); + if (target != nullptr) { + target->book.add_order( + add->order_reference_number, + to_side(add->buy_sell_indicator), + add->shares, + add->price + ); + emit_bbo_if_changed(*target); + } + return; + } + if (const auto* add = std::get_if(&message)) { + BookEntry* target = ensure_entry(add->stock_locate, to_string(add->stock, STOCK_LEN)); + if (target != nullptr) { + target->book.add_order( + add->order_reference_number, + to_side(add->buy_sell_indicator), + add->shares, + add->price + ); + emit_bbo_if_changed(*target); + } + return; + } + if (const auto* exec = std::get_if(&message)) { + BookEntry* target = entry(exec->stock_locate); + if (target == nullptr) { + return; + } + if (m_trade_callback) { + const auto price = target->book.order_price(exec->order_reference_number); + const auto side = target->book.order_side(exec->order_reference_number); + if (price.has_value()) { + Trade trade {}; + trade.timestamp = exec->timestamp; + trade.stock_locate = exec->stock_locate; + trade.symbol = target->book.symbol(); + trade.price = StandardPrice {*price}; + trade.shares = exec->executed_shares; + trade.match_number = exec->match_number; + trade.side = side.has_value() ? static_cast(*side) : '\0'; + trade.printable = true; + m_trade_callback(trade); + } + } + target->book.execute_order(exec->order_reference_number, exec->executed_shares); + emit_bbo_if_changed(*target); + return; + } + if (const auto* exec = std::get_if(&message)) { + BookEntry* target = entry(exec->stock_locate); + if (target == nullptr) { + return; + } + if (m_trade_callback) { + const auto side = target->book.order_side(exec->order_reference_number); + Trade trade {}; + trade.timestamp = exec->timestamp; + trade.stock_locate = exec->stock_locate; + trade.symbol = target->book.symbol(); + trade.price = StandardPrice {exec->execution_price}; + trade.shares = exec->executed_shares; + trade.match_number = exec->match_number; + trade.side = side.has_value() ? static_cast(*side) : '\0'; + trade.printable = exec->printable == 'Y'; + m_trade_callback(trade); + } + target->book.execute_order(exec->order_reference_number, exec->executed_shares); + emit_bbo_if_changed(*target); + return; + } + if (const auto* cancel = std::get_if(&message)) { + if (BookEntry* target = entry(cancel->stock_locate)) { + target->book.reduce_order(cancel->order_reference_number, cancel->cancelled_shares); + emit_bbo_if_changed(*target); + } + return; + } + if (const auto* del = std::get_if(&message)) { + if (BookEntry* target = entry(del->stock_locate)) { + target->book.delete_order(del->order_reference_number); + emit_bbo_if_changed(*target); + } + return; + } + if (const auto* replace = std::get_if(&message)) { + if (BookEntry* target = entry(replace->stock_locate)) { + target->book.replace_order( + replace->original_order_reference_number, + replace->new_order_reference_number, + replace->shares, + replace->price + ); + emit_bbo_if_changed(*target); + } + return; + } + if (const auto* trade = std::get_if(&message)) { + // A non-displayable order print does not alter the visible book. + if (m_trade_callback) { + Trade tape_entry {}; + tape_entry.timestamp = trade->timestamp; + tape_entry.stock_locate = trade->stock_locate; + tape_entry.symbol = to_string(trade->stock, STOCK_LEN); + tape_entry.price = StandardPrice {trade->price}; + tape_entry.shares = trade->shares; + tape_entry.match_number = trade->match_number; + tape_entry.side = trade->buy_sell_indicator; + tape_entry.printable = true; + m_trade_callback(tape_entry); + } + return; + } + if (const auto* cross = std::get_if(&message)) { + if (m_trade_callback) { + Trade tape_entry {}; + tape_entry.timestamp = cross->timestamp; + tape_entry.stock_locate = cross->stock_locate; + tape_entry.symbol = to_string(cross->stock, STOCK_LEN); + tape_entry.price = StandardPrice {cross->cross_price}; + tape_entry.shares = cross->shares; + tape_entry.match_number = cross->match_number; + tape_entry.printable = true; + tape_entry.is_cross = true; + tape_entry.cross_type = cross->cross_type; + m_trade_callback(tape_entry); + } + return; + } + if (const auto* directory = std::get_if(&message)) { + if (directory->stock_locate >= m_symbol_by_locate.size()) { + m_symbol_by_locate.resize(static_cast(directory->stock_locate) + 1); + } + m_symbol_by_locate[directory->stock_locate] = to_string(directory->stock, STOCK_LEN); + return; + } +} + +auto BookManager::book(std::uint16_t stock_locate) const -> const L3Book* { + const BookEntry* found = entry(stock_locate); + return found != nullptr ? &found->book : nullptr; +} + +auto BookManager::book_for_symbol(std::string_view symbol) const -> const L3Book* { + for (const auto& slot : m_books_by_locate) { + if (slot && slot->book.symbol() == symbol) { + return &slot->book; + } + } + return nullptr; +} + +auto BookManager::symbol_for_locate(std::uint16_t stock_locate) const -> std::string_view { + if (const BookEntry* found = entry(stock_locate); + found != nullptr && !found->book.symbol().empty()) { + return found->book.symbol(); + } + if (stock_locate < m_symbol_by_locate.size()) { + return m_symbol_by_locate[stock_locate]; + } + return {}; +} + +} // namespace itch::book diff --git a/src/book/l3_book.cpp b/src/book/l3_book.cpp new file mode 100644 index 0000000..e698ad7 --- /dev/null +++ b/src/book/l3_book.cpp @@ -0,0 +1,268 @@ +#include "itch/book/l3_book.hpp" + +#include +#include + +namespace itch::book { + +L3Book::L3Book(std::string symbol) : m_symbol {std::move(symbol)} {} + +auto L3Book::side_levels(Side side) noexcept -> std::vector& { + return side == Side::buy ? m_bids : m_asks; +} + +auto L3Book::side_levels(Side side) const noexcept -> const std::vector& { + return side == Side::buy ? m_bids : m_asks; +} + +auto L3Book::allocate_node() -> std::uint32_t { + if (m_free_head != NIL) { + const std::uint32_t index = m_free_head; + m_free_head = m_pool[index].next; + m_pool[index] = OrderNode {}; + return index; + } + m_pool.push_back(OrderNode {}); + return static_cast(m_pool.size() - 1); +} + +auto L3Book::free_node(std::uint32_t node_index) -> void { + m_pool[node_index] = OrderNode {}; + m_pool[node_index].next = m_free_head; + m_free_head = node_index; +} + +auto L3Book::find_level(Side side, std::uint32_t price) const -> std::uint32_t { + const auto& levels = side_levels(side); + // Bids are sorted descending, asks ascending; binary search accordingly. + if (side == Side::buy) { + const auto iter = std::lower_bound( + levels.begin(), levels.end(), price, [](const Level& level, std::uint32_t value) { + return level.price > value; + } + ); + if (iter != levels.end() && iter->price == price) { + return static_cast(iter - levels.begin()); + } + } else { + const auto iter = std::lower_bound( + levels.begin(), levels.end(), price, [](const Level& level, std::uint32_t value) { + return level.price < value; + } + ); + if (iter != levels.end() && iter->price == price) { + return static_cast(iter - levels.begin()); + } + } + return NIL; +} + +auto L3Book::find_or_create_level(Side side, std::uint32_t price) -> std::uint32_t { + auto& levels = side_levels(side); + auto position = + side == Side::buy + ? std::lower_bound( + levels.begin(), + levels.end(), + price, + [](const Level& level, std::uint32_t value) { return level.price > value; } + ) + : std::lower_bound( + levels.begin(), levels.end(), price, [](const Level& level, std::uint32_t value) { + return level.price < value; + } + ); + if (position != levels.end() && position->price == price) { + return static_cast(position - levels.begin()); + } + Level new_level {}; + new_level.price = price; + const auto inserted = levels.insert(position, new_level); + return static_cast(inserted - levels.begin()); +} + +auto L3Book::add_order( + std::uint64_t reference_number, Side side, std::uint32_t shares, std::uint32_t price +) -> void { + if (m_index.find(reference_number) != OrderIndex::NPOS) { + return; // Duplicate add; ignore to keep the book consistent. + } + const std::uint32_t node_index = allocate_node(); + OrderNode& node = m_pool[node_index]; + node.reference_number = reference_number; + node.shares = shares; + node.price = price; + node.side = side; + + const std::uint32_t level_index = find_or_create_level(side, price); + Level& level = side_levels(side)[level_index]; + + // Append to the tail of the level FIFO to preserve time priority. + node.prev = level.tail; + node.next = NIL; + if (level.tail != NIL) { + m_pool[level.tail].next = node_index; + } else { + level.head = node_index; + } + level.tail = node_index; + level.total_shares += shares; + ++level.order_count; + + m_index.insert(reference_number, node_index); +} + +auto L3Book::unlink_node(std::uint32_t node_index) -> void { + OrderNode& node = m_pool[node_index]; + const std::uint32_t level_index = find_level(node.side, node.price); + if (level_index == NIL) { + return; + } + auto& levels = side_levels(node.side); + Level& level = levels[level_index]; + + if (node.prev != NIL) { + m_pool[node.prev].next = node.next; + } else { + level.head = node.next; + } + if (node.next != NIL) { + m_pool[node.next].prev = node.prev; + } else { + level.tail = node.prev; + } + level.total_shares -= node.shares; + --level.order_count; + if (level.order_count == 0) { + levels.erase(levels.begin() + level_index); + } +} + +auto L3Book::execute_order(std::uint64_t reference_number, std::uint32_t shares) -> std::uint32_t { + const std::uint32_t node_index = m_index.find(reference_number); + if (node_index == OrderIndex::NPOS) { + return 0; + } + OrderNode& node = m_pool[node_index]; + const std::uint32_t removed = std::min(shares, node.shares); + + if (removed == node.shares) { + unlink_node(node_index); + free_node(node_index); + m_index.erase(reference_number); + return removed; + } + + node.shares -= removed; + const std::uint32_t level_index = find_level(node.side, node.price); + if (level_index != NIL) { + side_levels(node.side)[level_index].total_shares -= removed; + } + return removed; +} + +auto L3Book::reduce_order(std::uint64_t reference_number, std::uint32_t shares) -> std::uint32_t { + // A partial cancel removes shares exactly as an execution does. + return execute_order(reference_number, shares); +} + +auto L3Book::delete_order(std::uint64_t reference_number) -> void { + const std::uint32_t node_index = m_index.find(reference_number); + if (node_index == OrderIndex::NPOS) { + return; + } + unlink_node(node_index); + free_node(node_index); + m_index.erase(reference_number); +} + +auto L3Book::replace_order( + std::uint64_t old_reference_number, + std::uint64_t new_reference_number, + std::uint32_t shares, + std::uint32_t price +) -> void { + const std::uint32_t node_index = m_index.find(old_reference_number); + if (node_index == OrderIndex::NPOS) { + return; + } + const Side side = m_pool[node_index].side; + delete_order(old_reference_number); + add_order(new_reference_number, side, shares, price); +} + +auto L3Book::contains(std::uint64_t reference_number) const -> bool { + return m_index.contains(reference_number); +} + +auto L3Book::order_price(std::uint64_t reference_number) const -> std::optional { + const std::uint32_t node_index = m_index.find(reference_number); + if (node_index == OrderIndex::NPOS) { + return std::nullopt; + } + return m_pool[node_index].price; +} + +auto L3Book::order_side(std::uint64_t reference_number) const -> std::optional { + const std::uint32_t node_index = m_index.find(reference_number); + if (node_index == OrderIndex::NPOS) { + return std::nullopt; + } + return m_pool[node_index].side; +} + +auto L3Book::bbo() const -> Bbo { + Bbo result {}; + if (!m_bids.empty()) { + result.has_bid = true; + result.bid_price = StandardPrice {m_bids.front().price}; + result.bid_shares = m_bids.front().total_shares; + } + if (!m_asks.empty()) { + result.has_ask = true; + result.ask_price = StandardPrice {m_asks.front().price}; + result.ask_shares = m_asks.front().total_shares; + } + return result; +} + +auto L3Book::depth(Side side, std::size_t max_levels) const -> std::vector { + const auto& levels = side_levels(side); + const std::size_t count = max_levels == 0 ? levels.size() : std::min(max_levels, levels.size()); + std::vector result; + result.reserve(count); + for (std::size_t index = 0; index < count; ++index) { + result.push_back( + DepthLevel { + StandardPrice {levels[index].price}, + levels[index].total_shares, + levels[index].order_count + } + ); + } + return result; +} + +auto L3Book::orders_at(Side side, std::uint32_t price) const -> std::vector { + std::vector result; + const std::uint32_t level_index = find_level(side, price); + if (level_index == NIL) { + return result; + } + const Level& level = side_levels(side)[level_index]; + result.reserve(level.order_count); + for (std::uint32_t node_index = level.head; node_index != NIL; + node_index = m_pool[node_index].next) { + const OrderNode& node = m_pool[node_index]; + result.push_back( + OrderView {node.reference_number, node.shares, StandardPrice {node.price}} + ); + } + return result; +} + +auto L3Book::level_count(Side side) const noexcept -> std::size_t { + return side_levels(side).size(); +} + +} // namespace itch::book diff --git a/src/parser.cpp b/src/parser.cpp index 9a90e7b..3c74df2 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -9,6 +9,8 @@ #include #include +#include "itch/detail/wire.hpp" + namespace itch { namespace utils { @@ -228,23 +230,7 @@ auto unpack_message(DLCRMessage& msg, const char* buffer, size_t& offset) -> voi namespace { -// The on-wire ITCH timestamp is 48 bits (6 bytes), but every message struct -// stores it in a 64-bit field. Each struct therefore occupies exactly two bytes -// more than its on-wire encoding, since the timestamp is the only field whose -// storage width differs from its wire width. -constexpr std::size_t TIMESTAMP_STRUCT_PADDING = sizeof(std::uint64_t) - 6; - -/// @brief The exact on-wire size, in bytes, of a fully formed message of type T. -template -constexpr std::size_t WIRE_SIZE = sizeof(MsgType) - TIMESTAMP_STRUCT_PADDING; - -// Lock the padding assumption to the spec lengths so a future struct change -// that breaks the derivation is caught at compile time rather than at runtime. -static_assert(WIRE_SIZE == 12); -static_assert(WIRE_SIZE == 39); -static_assert(WIRE_SIZE == 36); -static_assert(WIRE_SIZE == 50); -static_assert(WIRE_SIZE == 48); +using detail::WIRE_SIZE; /// @brief Decodes the common header and type-specific body of a message of /// type MsgType from a frame, returning it wrapped in the Message variant. @@ -281,29 +267,9 @@ consteval auto build_dispatch_table() -> std::array(WIRE_SIZE), &decode_typed}; }; - add.operator()('S'); - add.operator()('R'); - add.operator()('H'); - add.operator()('Y'); - add.operator()('L'); - add.operator()('V'); - add.operator()('W'); - add.operator()('K'); - add.operator()('J'); - add.operator()('h'); - add.operator()('A'); - add.operator()('F'); - add.operator()('E'); - add.operator()('C'); - add.operator()('X'); - add.operator()('D'); - add.operator()('U'); - add.operator()('P'); - add.operator()('Q'); - add.operator()('B'); - add.operator()('I'); - add.operator()('N'); - add.operator()('O'); + // The canonical message-type registry lives in itch/detail/wire.hpp so the + // dispatch table, overlay, and encoder all share one definition. + detail::for_each_message_type(add); return table; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index dcd452a..fe942dd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -23,6 +23,8 @@ add_executable( test_price_time.cpp test_conformance.cpp transport/test_transport.cpp + book/test_book.cpp + book/test_overlay.cpp ) target_include_directories(itch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/tests/book/test_book.cpp b/tests/book/test_book.cpp new file mode 100644 index 0000000..25f5b62 --- /dev/null +++ b/tests/book/test_book.cpp @@ -0,0 +1,174 @@ +#include + +#include +#include +#include + +#include "itch/book/book_manager.hpp" +#include "itch/book/l3_book.hpp" +#include "itch/messages.hpp" + +namespace { + +using itch::book::L3Book; +using itch::book::Side; + +// Fills an 8-byte ITCH stock field from a symbol, space padded. +auto fill_stock(char (&dest)[itch::STOCK_LEN], std::string_view symbol) -> void { + std::memset(dest, ' ', itch::STOCK_LEN); + std::memcpy( + dest, symbol.data(), std::min(symbol.size(), static_cast(itch::STOCK_LEN)) + ); +} + +auto make_add( + std::uint16_t locate, + std::uint64_t ref, + char side, + std::uint32_t shares, + std::string_view symbol, + std::uint32_t price +) -> itch::AddOrderMessage { + itch::AddOrderMessage msg {}; + msg.stock_locate = locate; + msg.order_reference_number = ref; + msg.buy_sell_indicator = side; + msg.shares = shares; + fill_stock(msg.stock, symbol); + msg.price = price; + return msg; +} + +} // namespace + +TEST(L3Book, TracksBestBidOfferAcrossLevels) { + L3Book book {"AAPL"}; + book.add_order(1, Side::buy, 100, 1500000); + book.add_order(2, Side::buy, 200, 1500100); // better bid + book.add_order(3, Side::sell, 50, 1500300); + book.add_order(4, Side::sell, 75, 1500200); // better ask + + const auto bbo = book.bbo(); + ASSERT_TRUE(bbo.has_bid); + ASSERT_TRUE(bbo.has_ask); + EXPECT_EQ(bbo.bid_price.raw(), 1500100U); + EXPECT_EQ(bbo.bid_shares, 200U); + EXPECT_EQ(bbo.ask_price.raw(), 1500200U); + EXPECT_EQ(bbo.ask_shares, 75U); +} + +TEST(L3Book, AggregatesSharesAndKeepsTimePriority) { + L3Book book; + book.add_order(1, Side::buy, 100, 1000000); + book.add_order(2, Side::buy, 150, 1000000); // same level, later in queue + const auto depth = book.depth(Side::buy); + ASSERT_EQ(depth.size(), 1U); + EXPECT_EQ(depth.front().shares, 250U); + EXPECT_EQ(depth.front().order_count, 2U); + + const auto orders = book.orders_at(Side::buy, 1000000); + ASSERT_EQ(orders.size(), 2U); + EXPECT_EQ(orders[0].reference_number, 1U); // FIFO order preserved + EXPECT_EQ(orders[1].reference_number, 2U); +} + +TEST(L3Book, ExecutePartialThenFullRemovesOrder) { + L3Book book; + book.add_order(1, Side::sell, 100, 2000000); + EXPECT_EQ(book.execute_order(1, 40), 40U); + EXPECT_EQ(book.bbo().ask_shares, 60U); + EXPECT_EQ(book.execute_order(1, 999), 60U); // clamps to remaining + EXPECT_FALSE(book.contains(1)); + EXPECT_FALSE(book.bbo().has_ask); +} + +TEST(L3Book, CancelReducesAndDeleteRemoves) { + L3Book book; + book.add_order(1, Side::buy, 500, 1000000); + book.reduce_order(1, 200); + EXPECT_EQ(book.bbo().bid_shares, 300U); + book.delete_order(1); + EXPECT_TRUE(book.empty()); +} + +TEST(L3Book, ReplaceMovesOrderToNewPriceAndReference) { + L3Book book; + book.add_order(1, Side::buy, 100, 1000000); + book.replace_order(1, 2, 80, 1000500); + EXPECT_FALSE(book.contains(1)); + ASSERT_TRUE(book.contains(2)); + EXPECT_EQ(book.bbo().bid_price.raw(), 1000500U); + EXPECT_EQ(book.bbo().bid_shares, 80U); +} + +TEST(L3Book, HandlesCrossedBookState) { + L3Book book; + book.add_order(1, Side::buy, 100, 1000500); // bid above + book.add_order(2, Side::sell, 100, 1000000); // ask below -> crossed + const auto bbo = book.bbo(); + EXPECT_GT(bbo.bid_price.raw(), bbo.ask_price.raw()); // book represents the crossed state +} + +TEST(BookManager, RoutesMessagesToPerSymbolBooks) { + itch::book::BookManager manager; + manager.process(itch::Message {make_add(1, 10, 'B', 100, "AAPL", 1500000)}); + manager.process(itch::Message {make_add(2, 11, 'S', 200, "MSFT", 3000000)}); + + EXPECT_EQ(manager.book_count(), 2U); + const auto* apple = manager.book_for_symbol("AAPL"); + ASSERT_NE(apple, nullptr); + EXPECT_EQ(apple->bbo().bid_shares, 100U); + EXPECT_EQ(manager.book(2)->symbol(), "MSFT"); +} + +TEST(BookManager, UniverseFilterTracksOnlySelectedSymbols) { + itch::book::BookManager manager; + manager.track_symbol("AAPL"); + manager.process(itch::Message {make_add(1, 10, 'B', 100, "AAPL", 1500000)}); + manager.process(itch::Message {make_add(2, 11, 'S', 200, "MSFT", 3000000)}); + + EXPECT_EQ(manager.book_count(), 1U); + EXPECT_NE(manager.book_for_symbol("AAPL"), nullptr); + EXPECT_EQ(manager.book_for_symbol("MSFT"), nullptr); +} + +TEST(BookManager, EmitsBboEventsOnlyWhenTopChanges) { + itch::book::BookManager manager; + int bbo_events = 0; + manager.set_bbo_callback([&](const L3Book&, const itch::book::Bbo&) { ++bbo_events; }); + + manager.process(itch::Message {make_add(1, 10, 'B', 100, "AAPL", 1500000)}); // new top + manager.process( + itch::Message {make_add(1, 11, 'B', 100, "AAPL", 1400000)} + ); // worse, no change + EXPECT_EQ(bbo_events, 1); +} + +TEST(BookManager, ExtractsTradeTapeWithPrintableFlag) { + itch::book::BookManager manager; + std::vector tape; + manager.set_trade_callback([&](const itch::Trade& trade) { tape.push_back(trade); }); + + manager.process(itch::Message {make_add(1, 10, 'S', 100, "AAPL", 1500000)}); + + itch::OrderExecutedMessage exec {}; + exec.stock_locate = 1; + exec.order_reference_number = 10; + exec.executed_shares = 40; + exec.match_number = 999; + manager.process(itch::Message {exec}); + + itch::OrderExecutedWithPriceMessage exec_price {}; + exec_price.stock_locate = 1; + exec_price.order_reference_number = 10; + exec_price.executed_shares = 10; + exec_price.printable = 'N'; + exec_price.execution_price = 1499000; + manager.process(itch::Message {exec_price}); + + ASSERT_EQ(tape.size(), 2U); + EXPECT_EQ(tape[0].price.raw(), 1500000U); // E uses the resting order price + EXPECT_TRUE(tape[0].printable); + EXPECT_EQ(tape[1].price.raw(), 1499000U); // C uses the execution price + EXPECT_FALSE(tape[1].printable); // honours the printable flag +} diff --git a/tests/book/test_overlay.cpp b/tests/book/test_overlay.cpp new file mode 100644 index 0000000..4ff27c9 --- /dev/null +++ b/tests/book/test_overlay.cpp @@ -0,0 +1,83 @@ +#include + +#include +#include + +#include "itch/messages.hpp" +#include "itch/overlay.hpp" +#include "itch/parser.hpp" +#include "transport/frame_builders.hpp" + +namespace { + +// Concatenates several length-prefixed frames into one buffer. +auto build_buffer(const std::vector>& payloads) -> std::vector { + std::vector buffer; + for (const auto& payload : payloads) { + const auto frame = itch::test::length_prefixed(payload); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + return buffer; +} + +} // namespace + +TEST(Overlay, ViewFieldsMatchEagerDecode) { + const auto payload = itch::test::add_order_payload(7, 42, 'B', 500, "AAPL", 1500000); + const auto buffer = build_buffer({payload}); + + // Eager decode for the reference values. + itch::Parser parser; + std::vector messages = parser.parse(std::span {buffer}); + ASSERT_EQ(messages.size(), 1U); + const auto& eager = std::get(messages.front()); + + // Lazy overlay decode of the same bytes. + itch::overlay::AddOrderView view {}; + std::uint64_t count = itch::overlay::for_each_message( + std::span {buffer}, [&](const itch::overlay::MessageView& generic) { + view = itch::overlay::AddOrderView {generic.data(), generic.size()}; + } + ); + ASSERT_EQ(count, 1U); + + EXPECT_EQ(view.type(), 'A'); + EXPECT_EQ(view.stock_locate(), eager.stock_locate); + EXPECT_EQ(view.timestamp(), eager.timestamp); + EXPECT_EQ(view.order_reference_number(), eager.order_reference_number); + EXPECT_EQ(view.buy_sell_indicator(), eager.buy_sell_indicator); + EXPECT_EQ(view.shares(), eager.shares); + EXPECT_EQ(view.price(), eager.price); + EXPECT_EQ(itch::to_string(view.stock().data(), view.stock().size()), "AAPL"); +} + +TEST(Overlay, FramesMultipleMessagesAndSkipsUnknownTypes) { + std::vector> payloads = { + itch::test::add_order_payload(1, 1, 'B', 100, "AAA", 1000000), + itch::test::system_event_payload(5000, 'O'), + itch::test::add_order_payload(1, 2, 'S', 200, "AAA", 1000100), + }; + const auto buffer = build_buffer(payloads); + + std::uint64_t adds = 0; + const auto total = itch::overlay::for_each_message( + std::span {buffer}, [&](const itch::overlay::MessageView& view) { + if (view.type() == 'A') { + ++adds; + } + } + ); + EXPECT_EQ(total, 3U); + EXPECT_EQ(adds, 2U); +} + +TEST(Overlay, UndersizedFrameIsSkipped) { + // A frame claiming type 'A' but only 5 bytes long must not be delivered. + std::vector payload = { + std::byte {'A'}, std::byte {0}, std::byte {0}, std::byte {0}, std::byte {0} + }; + const auto buffer = build_buffer({payload}); + const auto total = + itch::overlay::for_each_message(std::span {buffer}, [](const auto&) {}); + EXPECT_EQ(total, 0U); +} diff --git a/tests/transport/frame_builders.hpp b/tests/transport/frame_builders.hpp index 2e20bdc..1fdb42b 100644 --- a/tests/transport/frame_builders.hpp +++ b/tests/transport/frame_builders.hpp @@ -59,6 +59,30 @@ inline auto system_event_payload(std::uint64_t timestamp, char event_code) return payload; } +/// @brief Builds the raw payload (no length prefix) of an Add Order (`A`) +/// message. +inline auto add_order_payload( + std::uint16_t locate, std::uint64_t ref, char side, std::uint32_t shares, + std::string_view symbol, std::uint32_t price +) -> std::vector { + std::vector payload; + payload.push_back(std::byte {'A'}); + append_be16(payload, locate); + append_be16(payload, 0); // tracking_number + std::vector ts; + append_be64(ts, 1234567); + payload.insert(payload.end(), ts.begin() + 2, ts.end()); // 48-bit timestamp + append_be64(payload, ref); + payload.push_back(std::byte {static_cast(side)}); + append_be32(payload, shares); + for (std::size_t index = 0; index < 8; ++index) { + const char chr = index < symbol.size() ? symbol[index] : ' '; + payload.push_back(std::byte {static_cast(chr)}); + } + append_be32(payload, price); + return payload; +} + /// @brief Wraps a raw payload as a length-prefixed ITCH frame (and message /// block, since both use the same 2-byte big-endian length prefix). inline auto length_prefixed(const std::vector& payload) -> std::vector { From 333de5866f1ab77ce966ff9b41e9c2803ddeaabf Mon Sep 17 00:00:00 2001 From: bbalouki Date: Mon, 29 Jun 2026 00:39:34 +0100 Subject: [PATCH 028/144] Add Phase 3 analytics & microstructure layer Implement the FEATURES.md Phase 3 header-only itch::analytics layer, driven off the Phase 2 trade tape and book: - bars.hpp: OHLCV BarBuilder over time/tick/volume clock policies. - vwap.hpp: running and interval VWAP and TWAP. - microstructure.hpp: spread, mid, depth-at-level, queue imbalance, and Cont-Kukanov-Stoikov order-flow imbalance. - imbalance.hpp: ImbalanceInfo surfacing NOII (I) data with strong price types. - auctions.hpp: AuctionTracker reconstructing opening/closing/halt/IPO crosses by pairing each Cross Trade (Q) with the latest NOII context. Adds deterministic analytics tests, a VWAP example, and README/CHANGELOG entries. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 16 ++ README.md | 34 +++++ VERSION.txt | 2 +- examples/CMakeLists.txt | 8 + examples/analytics/vwap_example.cpp | 41 +++++ include/itch/analytics/auctions.hpp | 98 ++++++++++++ include/itch/analytics/bars.hpp | 118 +++++++++++++++ include/itch/analytics/imbalance.hpp | 63 ++++++++ include/itch/analytics/microstructure.hpp | 91 ++++++++++++ include/itch/analytics/vwap.hpp | 88 +++++++++++ tests/CMakeLists.txt | 1 + tests/analytics/test_analytics.cpp | 173 ++++++++++++++++++++++ 12 files changed, 732 insertions(+), 1 deletion(-) create mode 100644 examples/analytics/vwap_example.cpp create mode 100644 include/itch/analytics/auctions.hpp create mode 100644 include/itch/analytics/bars.hpp create mode 100644 include/itch/analytics/imbalance.hpp create mode 100644 include/itch/analytics/microstructure.hpp create mode 100644 include/itch/analytics/vwap.hpp create mode 100644 tests/analytics/test_analytics.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e24bf6..3b48cae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Phase 3 — Analytics & microstructure** ([FEATURES.md](FEATURES.md)). A header-only + `itch::analytics` layer driven off the Phase 2 trade tape and book: + - Bar builders (`analytics/bars.hpp`): OHLCV aggregation over time, tick, and + volume clocks via a single `BarBuilder` template parameterized by a clock policy. + - VWAP and TWAP (`analytics/vwap.hpp`): running and interval volume- and + time-weighted average prices. + - Microstructure metrics (`analytics/microstructure.hpp`): spread, mid price, + depth-at-level, top-of-book queue imbalance, and Cont-Kukanov-Stoikov order-flow + imbalance. + - NOII / imbalance (`analytics/imbalance.hpp`): `ImbalanceInfo` surfacing the + `I` message fields with strong price types and a direction description. + - Auction reconstruction (`analytics/auctions.hpp`): `AuctionTracker` pairs each + Cross Trade (`Q`) with the latest NOII (`I`) context to reconstruct opening, + closing, halt, and IPO crosses. + - Analytics tests and a VWAP example (`examples/analytics/vwap_example.cpp`). + - **Phase 2 — Production order-book engine** ([FEATURES.md](FEATURES.md)). A full-market book engine alongside the existing single-symbol `LimitOrderBook` (which is unchanged and retained): diff --git a/README.md b/README.md index 156d2bd..651c7d7 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,9 @@ The design of this ITCH parser is guided by three principles: - **Zero-Copy Overlay API**: Inspect raw frames through lazy typed views that convert only the fields you read, for hot paths that touch a few fields per message. +- **Built-in Analytics**: Header-only microstructure layer — OHLCV bar builders + (time/tick/volume clocks), VWAP/TWAP, spread, queue imbalance, order-flow + imbalance, NOII surfacing, and auction reconstruction. See [Analytics](#analytics). - **High Throughput**: Multi-gigabyte-per-second parsing on modern hardware (see [Benchmarks](#benchmarks) for measured numbers). - **Allocation-Free Core**: The callback-based parsing loop performs zero dynamic memory allocations, minimizing latency and jitter. - **Type-Safe API**: All ITCH messages are deserialized into a `std::variant` of dedicated, packed `struct`s, ensuring compile-time safety. @@ -528,6 +531,37 @@ a zero-copy alternative to the eager parser: `for_each_message(buffer, cb)` yiel a `MessageView` (and typed views like `AddOrderView`) that decode each field lazily on access. +### Analytics + +The header-only `itch::analytics` layer computes the metrics quants ask for, +directly off the trade tape and book so every downstream team does not +reimplement them. + +| Component | Header | Provides | +| --------- | ------ | -------- | +| `BarBuilder` | `analytics/bars.hpp` | OHLCV bars over `TimeClock`, `TickClock`, `VolumeClock`. | +| `Vwap`, `Twap` | `analytics/vwap.hpp` | Running/interval volume- and time-weighted average price. | +| spread / mid / imbalance | `analytics/microstructure.hpp` | Spread, mid, depth-at-level, queue imbalance, order-flow imbalance. | +| `ImbalanceInfo` | `analytics/imbalance.hpp` | Decoded NOII (`I`) imbalance data. | +| `AuctionTracker` | `analytics/auctions.hpp` | Opening/closing/halt/IPO cross reconstruction. | + +```cpp +#include "itch/analytics/vwap.hpp" +#include "itch/analytics/bars.hpp" + +itch::analytics::Vwap vwap; +itch::analytics::BarBuilder bars{itch::analytics::TimeClock{60'000'000'000ULL}, // 1-minute bars + [](const itch::analytics::Bar& bar) { /* ... */ }}; + +manager.set_trade_callback([&](const itch::Trade& trade) { + vwap.add(trade.price, trade.shares); + bars.add(trade); +}); +// ... parse the feed ... +bars.flush(); +double session_vwap = vwap.value(); +``` + --- ## API Reference: ITCH 5.0 Message Types diff --git a/VERSION.txt b/VERSION.txt index f0bb29e..88c5fb8 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.3.0 +1.4.0 diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 60a4e57..cafc608 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -31,3 +31,11 @@ target_link_libraries( itch::itch warnings::strict ) + +add_executable(vwap_example analytics/vwap_example.cpp) + +target_link_libraries( + vwap_example PRIVATE + itch::itch + warnings::strict +) diff --git a/examples/analytics/vwap_example.cpp b/examples/analytics/vwap_example.cpp new file mode 100644 index 0000000..cf596a5 --- /dev/null +++ b/examples/analytics/vwap_example.cpp @@ -0,0 +1,41 @@ +#include +#include +#include + +#include "itch/analytics/vwap.hpp" +#include "itch/book/book_manager.hpp" +#include "itch/parser.hpp" + +// Computes the session VWAP for a chosen symbol by driving the BookManager's +// trade tape through a running Vwap accumulator. +auto main(int argc, char* argv[]) -> int { + if (argc < 3) { + std::println(stderr, "Usage: {} ", argv[0]); + return 1; + } + + std::ifstream file {argv[1], std::ios::binary}; + if (!file) { + std::println(stderr, "Error: cannot open '{}'.", argv[1]); + return 1; + } + std::vector buffer {std::istreambuf_iterator {file}, {}}; + + itch::book::BookManager manager; + manager.track_symbol(argv[2]); + + itch::analytics::Vwap vwap; + manager.set_trade_callback([&](const itch::Trade& trade) { + if (trade.symbol == argv[2]) { + vwap.add(trade.price, trade.shares); + } + }); + + itch::Parser parser; + parser.parse(buffer.data(), buffer.size(), [&](const itch::Message& msg) { + manager.process(msg); + }); + + std::println("VWAP for {}: {:.4f} over {} shares", argv[2], vwap.value(), vwap.volume()); + return 0; +} diff --git a/include/itch/analytics/auctions.hpp b/include/itch/analytics/auctions.hpp new file mode 100644 index 0000000..5e55df3 --- /dev/null +++ b/include/itch/analytics/auctions.hpp @@ -0,0 +1,98 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/analytics/imbalance.hpp" +#include "itch/indicators.hpp" +#include "itch/messages.hpp" +#include "itch/price.hpp" + +namespace itch::analytics { + +/// @brief A reconstructed auction (cross) event. +/// +/// Nasdaq runs the opening, closing, halt, and IPO crosses; their price discovery +/// is disseminated through the NOII (`I`) messages leading up to the cross and the +/// Cross Trade (`Q`) message that prints it. This record pairs the executed cross +/// with the imbalance state that immediately preceded it. +struct Auction { + std::uint64_t timestamp {0}; + std::uint16_t stock_locate {0}; + std::string stock; + char cross_type {'\0'}; ///< 'O' open, 'C' close, 'H' halt/pause, 'I' IPO. + StandardPrice cross_price {}; ///< The cross execution price. + std::uint64_t cross_shares {0}; ///< Shares executed in the cross. + std::uint64_t paired_shares {0}; ///< Paired shares from the latest NOII. + std::uint64_t imbalance_shares {0}; ///< Imbalance shares from the latest NOII. + char imbalance_direction {'\0'}; + bool had_imbalance {false}; ///< Whether NOII context was available. +}; + +/// @brief A human-readable description of a cross type code. +[[nodiscard]] inline auto cross_type_name(char cross_type) -> std::string_view { + constexpr indicators::FrozenMap types {std::to_array>({ + {'O', "Opening Cross"}, + {'C', "Closing Cross"}, + {'H', "Cross for halted/paused security"}, + {'I', "Intraday/IPO Cross"}, + })}; + return types.at_or(cross_type, "Unknown"); +} + +/// @brief Reconstructs auctions from the NOII and Cross Trade message stream. +/// +/// Feed every message with `process`: NOII (`I`) messages update the latest +/// imbalance state per security, and a Cross Trade (`Q`) finalizes an `Auction` +/// using the executed cross price together with that stored imbalance context, +/// which is delivered to the auction callback. +class AuctionTracker { + public: + using AuctionCallback = std::function; + + /// @brief Installs the callback invoked for each reconstructed auction. + auto set_auction_callback(AuctionCallback callback) -> void { + m_callback = std::move(callback); + } + + /// @brief Processes one ITCH message, emitting an auction on each cross. + auto process(const Message& message) -> void { + if (const auto* noii = std::get_if(&message)) { + m_latest_imbalance[noii->stock_locate] = make_imbalance_info(*noii); + return; + } + if (const auto* cross = std::get_if(&message)) { + Auction auction {}; + auction.timestamp = cross->timestamp; + auction.stock_locate = cross->stock_locate; + auction.stock = to_string(cross->stock, STOCK_LEN); + auction.cross_type = cross->cross_type; + auction.cross_price = StandardPrice {cross->cross_price}; + auction.cross_shares = cross->shares; + + const auto iter = m_latest_imbalance.find(cross->stock_locate); + if (iter != m_latest_imbalance.end()) { + auction.paired_shares = iter->second.paired_shares; + auction.imbalance_shares = iter->second.imbalance_shares; + auction.imbalance_direction = iter->second.imbalance_direction; + auction.had_imbalance = true; + } + if (m_callback) { + m_callback(auction); + } + return; + } + } + + private: + std::unordered_map m_latest_imbalance; + AuctionCallback m_callback {}; +}; + +} // namespace itch::analytics diff --git a/include/itch/analytics/bars.hpp b/include/itch/analytics/bars.hpp new file mode 100644 index 0000000..e40efca --- /dev/null +++ b/include/itch/analytics/bars.hpp @@ -0,0 +1,118 @@ +#pragma once + +#include +#include +#include +#include + +#include "itch/price.hpp" +#include "itch/tape.hpp" + +namespace itch::analytics { + +/// @brief One OHLCV bar aggregated from the trade tape. +struct Bar { + std::uint64_t bucket {0}; ///< Clock bucket id this bar covers. + std::uint64_t start_timestamp {0}; ///< Timestamp of the first trade in the bar. + std::uint64_t end_timestamp {0}; ///< Timestamp of the last trade in the bar. + StandardPrice open {}; ///< First trade price. + StandardPrice high {}; ///< Highest trade price. + StandardPrice low {}; ///< Lowest trade price. + StandardPrice close {}; ///< Last trade price. + std::uint64_t volume {0}; ///< Total shares traded in the bar. + std::uint64_t trade_count {0}; ///< Number of trades in the bar. +}; + +/// @brief Callback invoked with each completed bar. +using BarCallback = std::function; + +/// @brief A clock that buckets trades by wall-clock time (nanoseconds). +struct TimeClock { + std::uint64_t interval_ns {1}; + [[nodiscard]] auto bucket(const Trade& trade, std::uint64_t, std::uint64_t) const noexcept + -> std::uint64_t { + return trade.timestamp / interval_ns; + } +}; + +/// @brief A clock that buckets a fixed number of trades into each bar. +struct TickClock { + std::uint64_t ticks_per_bar {1}; + [[nodiscard]] auto bucket(const Trade&, std::uint64_t, std::uint64_t tick_index) const noexcept + -> std::uint64_t { + return tick_index / ticks_per_bar; + } +}; + +/// @brief A clock that buckets a fixed amount of traded volume into each bar. +struct VolumeClock { + std::uint64_t volume_per_bar {1}; + [[nodiscard]] auto bucket( + const Trade&, std::uint64_t cumulative_volume, std::uint64_t + ) const noexcept -> std::uint64_t { + return cumulative_volume / volume_per_bar; + } +}; + +/// @brief Builds OHLCV bars from a trade stream under a configurable clock. +/// +/// Feed trades in timestamp order with `add`; each time the clock's bucket id +/// changes, the completed bar is emitted to the callback. `flush` emits the final, +/// partial bar. The clock is a small policy (see `TimeClock`, `TickClock`, +/// `VolumeClock`) so time-, tick-, and volume-based aggregation share one builder. +template +class BarBuilder { + public: + BarBuilder(Clock clock, BarCallback callback) + : m_clock {std::move(clock)}, m_callback {std::move(callback)} {} + + /// @brief Adds one trade, emitting the previous bar when the bucket changes. + auto add(const Trade& trade) -> void { + const std::uint64_t bucket = m_clock.bucket(trade, m_cumulative_volume, m_tick_index); + if (m_has_bar && bucket != m_bar.bucket) { + emit(); + } + if (!m_has_bar) { + m_bar = Bar {}; + m_bar.bucket = bucket; + m_bar.start_timestamp = trade.timestamp; + m_bar.open = trade.price; + m_bar.high = trade.price; + m_bar.low = trade.price; + m_has_bar = true; + } + m_bar.high = std::max(m_bar.high, trade.price); + m_bar.low = std::min(m_bar.low, trade.price); + m_bar.close = trade.price; + m_bar.end_timestamp = trade.timestamp; + m_bar.volume += trade.shares; + ++m_bar.trade_count; + + m_cumulative_volume += trade.shares; + ++m_tick_index; + } + + /// @brief Emits the current partial bar, if any, and clears it. + auto flush() -> void { + if (m_has_bar) { + emit(); + } + } + + private: + auto emit() -> void { + if (m_callback) { + m_callback(m_bar); + } + m_has_bar = false; + } + + Clock m_clock; + BarCallback m_callback; + Bar m_bar {}; + bool m_has_bar {false}; + std::uint64_t m_cumulative_volume {0}; + std::uint64_t m_tick_index {0}; +}; + +} // namespace itch::analytics diff --git a/include/itch/analytics/imbalance.hpp b/include/itch/analytics/imbalance.hpp new file mode 100644 index 0000000..75261e7 --- /dev/null +++ b/include/itch/analytics/imbalance.hpp @@ -0,0 +1,63 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "itch/indicators.hpp" +#include "itch/messages.hpp" +#include "itch/price.hpp" + +namespace itch::analytics { + +/// @brief A usable, decoded view of a Net Order Imbalance Indicator (`I`) message. +/// +/// The feed already carries the NOII data used during the opening and closing +/// crosses; this surfaces it with strong price types and a trimmed symbol instead +/// of the raw packed fields. +struct ImbalanceInfo { + std::uint64_t timestamp {0}; + std::uint16_t stock_locate {0}; + std::string stock; + std::uint64_t paired_shares {0}; + std::uint64_t imbalance_shares {0}; + char imbalance_direction {'\0'}; + StandardPrice far_price {}; + StandardPrice near_price {}; + StandardPrice current_reference_price {}; + char cross_type {'\0'}; + char price_variation_indicator {'\0'}; +}; + +/// @brief Builds an `ImbalanceInfo` from a raw NOII message. +[[nodiscard]] inline auto make_imbalance_info(const NOIIMessage& msg) -> ImbalanceInfo { + ImbalanceInfo info {}; + info.timestamp = msg.timestamp; + info.stock_locate = msg.stock_locate; + info.stock = to_string(msg.stock, STOCK_LEN); + info.paired_shares = msg.paired_shares; + info.imbalance_shares = msg.imbalance_shares; + info.imbalance_direction = msg.imbalance_direction; + info.far_price = StandardPrice {msg.far_price}; + info.near_price = StandardPrice {msg.near_price}; + info.current_reference_price = StandardPrice {msg.current_reference_price}; + info.cross_type = msg.cross_type; + info.price_variation_indicator = msg.price_variation_indicator; + return info; +} + +/// @brief A human-readable description of an imbalance direction code. +[[nodiscard]] inline auto imbalance_direction_name(char direction) -> std::string_view { + constexpr indicators::FrozenMap directions {std::to_array>({ + {'B', "Buy imbalance"}, + {'S', "Sell imbalance"}, + {'N', "No imbalance"}, + {'O', "Insufficient orders to calculate"}, + {'P', "Paused"}, + })}; + return directions.at_or(direction, "Unknown"); +} + +} // namespace itch::analytics diff --git a/include/itch/analytics/microstructure.hpp b/include/itch/analytics/microstructure.hpp new file mode 100644 index 0000000..6650676 --- /dev/null +++ b/include/itch/analytics/microstructure.hpp @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include + +#include "itch/book/l3_book.hpp" + +namespace itch::analytics { + +/// @brief The bid-ask spread in price units, or NaN if either side is empty. +[[nodiscard]] inline auto spread(const book::Bbo& bbo) -> double { + if (!bbo.has_bid || !bbo.has_ask) { + return std::numeric_limits::quiet_NaN(); + } + return bbo.ask_price.to_double() - bbo.bid_price.to_double(); +} + +/// @brief The mid price ((bid + ask) / 2), or NaN if either side is empty. +[[nodiscard]] inline auto mid_price(const book::Bbo& bbo) -> double { + if (!bbo.has_bid || !bbo.has_ask) { + return std::numeric_limits::quiet_NaN(); + } + return (bbo.bid_price.to_double() + bbo.ask_price.to_double()) / 2.0; +} + +/// @brief Top-of-book queue imbalance in [-1, 1]. +/// +/// Positive values indicate more size resting on the bid than the offer +/// (buy-side pressure); negative values the reverse. Returns NaN when there is no +/// resting size at the top of either side. +[[nodiscard]] inline auto queue_imbalance(const book::Bbo& bbo) -> double { + const double bid = static_cast(bbo.bid_shares); + const double ask = static_cast(bbo.ask_shares); + const double total = bid + ask; + if (total <= 0.0) { + return std::numeric_limits::quiet_NaN(); + } + return (bid - ask) / total; +} + +/// @brief Total displayed shares within the best `levels` price levels of a side. +[[nodiscard]] inline auto depth_at_level( + const book::L3Book& book, book::Side side, std::size_t levels +) -> std::uint64_t { + std::uint64_t total = 0; + for (const auto& level : book.depth(side, levels)) { + total += level.shares; + } + return total; +} + +/// @brief The order-flow imbalance between two consecutive BBO observations. +/// +/// Implements the standard order-flow-imbalance contribution of Cont, Kukanov and +/// Stoikov: each side compares the new best price and size against the previous to +/// attribute added or removed liquidity, and the offer contribution is subtracted +/// from the bid contribution. Positive values indicate net buy-side flow. +[[nodiscard]] inline auto order_flow_imbalance(const book::Bbo& previous, const book::Bbo& current) + -> double { + const double prev_bid_price = previous.bid_price.to_double(); + const double curr_bid_price = current.bid_price.to_double(); + const double prev_ask_price = previous.ask_price.to_double(); + const double curr_ask_price = current.ask_price.to_double(); + const double prev_bid_size = static_cast(previous.bid_shares); + const double curr_bid_size = static_cast(current.bid_shares); + const double prev_ask_size = static_cast(previous.ask_shares); + const double curr_ask_size = static_cast(current.ask_shares); + + double bid_flow = 0.0; + if (curr_bid_price > prev_bid_price) { + bid_flow = curr_bid_size; + } else if (curr_bid_price == prev_bid_price) { + bid_flow = curr_bid_size - prev_bid_size; + } else { + bid_flow = -prev_bid_size; + } + + double ask_flow = 0.0; + if (curr_ask_price > prev_ask_price) { + ask_flow = prev_ask_size; + } else if (curr_ask_price == prev_ask_price) { + ask_flow = curr_ask_size - prev_ask_size; + } else { + ask_flow = -curr_ask_size; + } + + return bid_flow - ask_flow; +} + +} // namespace itch::analytics diff --git a/include/itch/analytics/vwap.hpp b/include/itch/analytics/vwap.hpp new file mode 100644 index 0000000..668de29 --- /dev/null +++ b/include/itch/analytics/vwap.hpp @@ -0,0 +1,88 @@ +#pragma once + +#include +#include + +#include "itch/price.hpp" + +namespace itch::analytics { + +/// @brief Running volume-weighted average price. +/// +/// Accumulates the sum of price times shares and the total shares; the value is +/// their ratio. Use `reset` at the start of each interval for an interval VWAP. +class Vwap { + public: + /// @brief Adds an execution of `shares` at `price` to the accumulation. + auto add(StandardPrice price, std::uint64_t shares) -> void { + m_price_volume += price.to_double() * static_cast(shares); + m_volume += shares; + } + + /// @brief The current VWAP, or NaN when no volume has been added. + [[nodiscard]] auto value() const -> double { + if (m_volume == 0) { + return std::numeric_limits::quiet_NaN(); + } + return m_price_volume / static_cast(m_volume); + } + + /// @brief The total shares accumulated so far. + [[nodiscard]] auto volume() const noexcept -> std::uint64_t { return m_volume; } + + /// @brief Clears the accumulation (start a new interval). + auto reset() noexcept -> void { + m_price_volume = 0.0; + m_volume = 0; + } + + private: + double m_price_volume {0.0}; + std::uint64_t m_volume {0}; +}; + +/// @brief Running time-weighted average price. +/// +/// Integrates the prevailing price over time: each sample contributes its price +/// weighted by the elapsed time since the previous sample. The value is the +/// time-weighted mean over the observed span. +class Twap { + public: + /// @brief Records that the price became `price` at `timestamp` (ns). + auto add(StandardPrice price, std::uint64_t timestamp) -> void { + if (m_has_sample && timestamp > m_last_timestamp) { + const double elapsed = static_cast(timestamp - m_last_timestamp); + m_price_time += m_last_price * elapsed; + m_total_time += elapsed; + } + m_last_price = price.to_double(); + m_last_timestamp = timestamp; + m_has_sample = true; + } + + /// @brief The current TWAP, or NaN when no span has elapsed. + [[nodiscard]] auto value() const -> double { + if (m_total_time <= 0.0) { + return m_has_sample ? m_last_price : std::numeric_limits::quiet_NaN(); + } + return m_price_time / m_total_time; + } + + /// @brief Clears the accumulation (start a new interval). + auto reset() noexcept -> void { + m_price_time = 0.0; + m_total_time = 0.0; + m_last_price = 0.0; + m_last_timestamp = 0; + m_has_sample = false; + } + + private: + double m_price_time {0.0}; + double m_total_time {0.0}; + double m_last_price {0.0}; + std::uint64_t m_last_timestamp {0}; + bool m_has_sample {false}; +}; + +} // namespace itch::analytics diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fe942dd..6281f70 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,6 +25,7 @@ add_executable( transport/test_transport.cpp book/test_book.cpp book/test_overlay.cpp + analytics/test_analytics.cpp ) target_include_directories(itch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/tests/analytics/test_analytics.cpp b/tests/analytics/test_analytics.cpp new file mode 100644 index 0000000..81714f5 --- /dev/null +++ b/tests/analytics/test_analytics.cpp @@ -0,0 +1,173 @@ +#include + +#include +#include +#include + +#include "itch/analytics/auctions.hpp" +#include "itch/analytics/bars.hpp" +#include "itch/analytics/imbalance.hpp" +#include "itch/analytics/microstructure.hpp" +#include "itch/analytics/vwap.hpp" +#include "itch/book/l3_book.hpp" +#include "itch/messages.hpp" +#include "itch/tape.hpp" + +namespace { + +auto make_trade(std::uint64_t timestamp, std::uint32_t raw_price, std::uint64_t shares) + -> itch::Trade { + itch::Trade trade {}; + trade.timestamp = timestamp; + trade.price = itch::StandardPrice {raw_price}; + trade.shares = shares; + return trade; +} + +auto fill_stock(char (&dest)[itch::STOCK_LEN], std::string_view symbol) -> void { + std::memset(dest, ' ', itch::STOCK_LEN); + std::memcpy( + dest, symbol.data(), std::min(symbol.size(), static_cast(itch::STOCK_LEN)) + ); +} + +} // namespace + +TEST(Bars, TimeClockGroupsTradesIntoIntervals) { + std::vector bars; + itch::analytics::BarBuilder builder { + itch::analytics::TimeClock {10}, + [&](const itch::analytics::Bar& bar) { bars.push_back(bar); } + }; + + builder.add(make_trade(1, 100, 5)); // bucket 0 + builder.add(make_trade(5, 120, 3)); // bucket 0 + builder.add(make_trade(12, 90, 4)); // bucket 1 -> closes bucket 0 + builder.flush(); + + ASSERT_EQ(bars.size(), 2U); + EXPECT_EQ(bars[0].open.raw(), 100U); + EXPECT_EQ(bars[0].high.raw(), 120U); + EXPECT_EQ(bars[0].low.raw(), 100U); + EXPECT_EQ(bars[0].close.raw(), 120U); + EXPECT_EQ(bars[0].volume, 8U); + EXPECT_EQ(bars[0].trade_count, 2U); + EXPECT_EQ(bars[1].open.raw(), 90U); + EXPECT_EQ(bars[1].volume, 4U); +} + +TEST(Bars, VolumeClockClosesBarsOnVolumeThreshold) { + std::vector bars; + itch::analytics::BarBuilder builder { + itch::analytics::VolumeClock {100}, + [&](const itch::analytics::Bar& bar) { bars.push_back(bar); } + }; + builder.add(make_trade(1, 100, 60)); // cumulative 0 -> bucket 0 + builder.add(make_trade(2, 101, 50)); // cumulative 60 -> bucket 0 (now 110) + builder.add(make_trade(3, 102, 10)); // cumulative 110 -> bucket 1 -> closes bar 0 + builder.flush(); + ASSERT_EQ(bars.size(), 2U); + EXPECT_EQ(bars[0].volume, 110U); +} + +TEST(Vwap, ComputesVolumeWeightedAverage) { + itch::analytics::Vwap vwap; + vwap.add(itch::StandardPrice {100}, 10); + vwap.add(itch::StandardPrice {200}, 30); + // (100*10 + 200*30) / 40 = 175 raw -> /10000 = 0.0175 + EXPECT_DOUBLE_EQ(vwap.value(), 175.0 / 10000.0); + EXPECT_EQ(vwap.volume(), 40U); +} + +TEST(Twap, ComputesTimeWeightedAverage) { + itch::analytics::Twap twap; + twap.add(itch::StandardPrice {10000}, 0); // price 1.0 from t=0 + twap.add(itch::StandardPrice {30000}, 10); // 1.0 held for 10 units, then 3.0 + twap.add(itch::StandardPrice {30000}, 20); // 3.0 held for 10 units + // (1.0*10 + 3.0*10) / 20 = 2.0 + EXPECT_DOUBLE_EQ(twap.value(), 2.0); +} + +TEST(Microstructure, SpreadMidAndQueueImbalance) { + itch::book::Bbo bbo {}; + bbo.has_bid = true; + bbo.has_ask = true; + bbo.bid_price = itch::StandardPrice {1000000}; + bbo.ask_price = itch::StandardPrice {1000200}; + bbo.bid_shares = 300; + bbo.ask_shares = 100; + + EXPECT_NEAR(itch::analytics::spread(bbo), 0.02, 1e-9); + EXPECT_NEAR(itch::analytics::mid_price(bbo), 100.01, 1e-9); + EXPECT_DOUBLE_EQ(itch::analytics::queue_imbalance(bbo), 0.5); // (300-100)/400 +} + +TEST(Microstructure, OrderFlowImbalanceFollowsContModel) { + itch::book::Bbo previous {}; + previous.bid_price = itch::StandardPrice {1000000}; + previous.ask_price = itch::StandardPrice {1000200}; + previous.bid_shares = 100; + previous.ask_shares = 100; + + itch::book::Bbo current = previous; + current.bid_shares = 150; // size added at unchanged best bid -> +50 buy flow + EXPECT_DOUBLE_EQ(itch::analytics::order_flow_imbalance(previous, current), 50.0); +} + +TEST(Microstructure, DepthAtLevelSumsBestLevels) { + itch::book::L3Book book; + book.add_order(1, itch::book::Side::buy, 100, 1000000); + book.add_order(2, itch::book::Side::buy, 200, 999900); + book.add_order(3, itch::book::Side::buy, 300, 999800); + EXPECT_EQ(itch::analytics::depth_at_level(book, itch::book::Side::buy, 2), 300U); + EXPECT_EQ(itch::analytics::depth_at_level(book, itch::book::Side::buy, 0), 600U); +} + +TEST(Imbalance, SurfacesNoiiFields) { + itch::NOIIMessage msg {}; + msg.stock_locate = 5; + msg.timestamp = 42; + fill_stock(msg.stock, "AAPL"); + msg.paired_shares = 1000; + msg.imbalance_shares = 250; + msg.imbalance_direction = 'B'; + msg.current_reference_price = 1500000; + + const auto info = itch::analytics::make_imbalance_info(msg); + EXPECT_EQ(info.stock, "AAPL"); + EXPECT_EQ(info.paired_shares, 1000U); + EXPECT_EQ(info.imbalance_shares, 250U); + EXPECT_EQ(info.current_reference_price.raw(), 1500000U); + EXPECT_EQ(itch::analytics::imbalance_direction_name('B'), "Buy imbalance"); +} + +TEST(Auctions, ReconstructsCrossWithImbalanceContext) { + itch::analytics::AuctionTracker tracker; + std::vector auctions; + tracker.set_auction_callback([&](const itch::analytics::Auction& auction) { + auctions.push_back(auction); + }); + + itch::NOIIMessage noii {}; + noii.stock_locate = 7; + fill_stock(noii.stock, "MSFT"); + noii.paired_shares = 5000; + noii.imbalance_shares = 1200; + noii.imbalance_direction = 'S'; + tracker.process(itch::Message {noii}); + + itch::CrossTradeMessage cross {}; + cross.stock_locate = 7; + fill_stock(cross.stock, "MSFT"); + cross.shares = 6200; + cross.cross_price = 3000000; + cross.cross_type = 'O'; + tracker.process(itch::Message {cross}); + + ASSERT_EQ(auctions.size(), 1U); + EXPECT_EQ(auctions[0].cross_price.raw(), 3000000U); + EXPECT_EQ(auctions[0].cross_type, 'O'); + EXPECT_TRUE(auctions[0].had_imbalance); + EXPECT_EQ(auctions[0].imbalance_shares, 1200U); + EXPECT_EQ(itch::analytics::cross_type_name('O'), "Opening Cross"); +} From fbd6cc02e69dfdebb9b8e02ed308f10c85130a7d Mon Sep 17 00:00:00 2001 From: bbalouki Date: Mon, 29 Jun 2026 01:26:00 +0100 Subject: [PATCH 029/144] Add Phase 4 research & interoperability Implement the FEATURES.md Phase 4 layer that meets quants in their own tools: - io/sink.hpp + io/csv_sink.hpp: a generic MessageSink and a dependency-free CsvSink flattening every message type into a normalized wide CSV table. - io/arrow_export.hpp + io/arrow_export.cpp: Apache Arrow / Parquet columnar export, gated behind the optional ITCH_WITH_ARROW CMake option (arrow vcpkg feature); off by default so the core stays dependency-free. - tools/itch_tool: the itch-tool CLI (stats/inspect/filter/convert), gated by ITCH_BUILD_TOOLS, auto-detecting raw ITCH vs .pcap/.pcapng input. - python/: pybind11 bindings (ITCH_BUILD_PYTHON) mirroring the pure-Python `itch` package API (MessageParser, typed messages, decode_price) as a faster drop-in backend, plus the book engine and VWAP; with pyproject.toml and pytest tests. - Per-message Doxygen documentation for every message struct and field in messages.hpp, sourced from the itch package's message semantics. Adds CSV unit test, vcpkg python/arrow features, and CI jobs for tools, examples, and the Python bindings. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/build.yml | 50 +++ CHANGELOG.md | 19 ++ CMakeLists.txt | 6 + README.md | 34 ++ VERSION.txt | 2 +- cmake/Helpers.cmake | 3 + include/itch/io/arrow_export.hpp | 62 ++++ include/itch/io/csv_sink.hpp | 41 +++ include/itch/io/sink.hpp | 28 ++ include/itch/messages.hpp | 533 ++++++++++++++++++++----------- python/CMakeLists.txt | 15 + python/README.md | 55 ++++ python/pyproject.toml | 28 ++ python/src/bindings.cpp | 392 +++++++++++++++++++++++ python/tests/test_bindings.py | 67 ++++ src/CMakeLists.txt | 13 + src/io/arrow_export.cpp | 257 +++++++++++++++ src/io/csv_sink.cpp | 141 ++++++++ tests/CMakeLists.txt | 1 + tests/io/test_csv_sink.cpp | 46 +++ tools/itch_tool/CMakeLists.txt | 14 + tools/itch_tool/main.cpp | 174 ++++++++++ vcpkg.json | 19 ++ 23 files changed, 1808 insertions(+), 192 deletions(-) create mode 100644 include/itch/io/arrow_export.hpp create mode 100644 include/itch/io/csv_sink.hpp create mode 100644 include/itch/io/sink.hpp create mode 100644 python/CMakeLists.txt create mode 100644 python/README.md create mode 100644 python/pyproject.toml create mode 100644 python/src/bindings.cpp create mode 100644 python/tests/test_bindings.py create mode 100644 src/io/arrow_export.cpp create mode 100644 src/io/csv_sink.cpp create mode 100644 tests/io/test_csv_sink.cpp create mode 100644 tools/itch_tool/CMakeLists.txt create mode 100644 tools/itch_tool/main.cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4e0e87b..f920265 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,6 +66,8 @@ jobs: cmake -S . -B $BUILD_DIR \ -DITCH_PROJECT_ENV=PROD \ -DITCH_BUILD_TESTS=ON \ + -DITCH_BUILD_TOOLS=ON \ + -DITCH_BUILD_EXAMPLES=ON \ -DITCH_CXX_STANDARD=${{ matrix.cpp_standard }} \ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ @@ -262,6 +264,54 @@ jobs: - name: Run transport fuzzer (time-budgeted) run: $BUILD_DIR/fuzz/moldudp64_fuzzer -max_total_time=60 -max_len=2048 -print_final_stats=1 + python-bindings: + runs-on: ubuntu-latest + env: + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + ${{ env.VCPKG_ROOT }} + ~/.cache/vcpkg + key: ${{ runner.os }}-vcpkg-python-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg- + + - name: Set up vcpkg + uses: lukka/run-vcpkg@v11 + + - name: Configure CMake (Python bindings) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_PYTHON=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython_EXECUTABLE=$(which python) \ + -DVCPKG_MANIFEST_FEATURES=python \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-linux + + - name: Build extension + run: cmake --build $BUILD_DIR --target itchcpp_python + + - name: Run binding tests + run: | + python -m pip install --upgrade pip pytest + cp $BUILD_DIR/python/itchcpp*.so python/tests/ + cd python/tests && python -m pytest -q + benchmark-smoke: runs-on: ubuntu-latest env: diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b48cae..7830155 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Phase 4 — Research & interoperability** ([FEATURES.md](FEATURES.md)): + - CSV and streaming sinks (`itch/io/sink.hpp`, `itch/io/csv_sink.hpp`): a generic + `MessageSink` interface and a `CsvSink` that flattens every message type into a + normalized wide CSV table, with no dependencies. + - Apache Arrow / Parquet export (`itch/io/arrow_export.hpp`), gated behind the + optional `ITCH_WITH_ARROW` CMake option (Arrow via the `arrow` vcpkg feature); + turns a feed into a columnar table for pandas/Polars/DuckDB/Spark. Off by + default so the core stays dependency-free. + - `itch-tool` command-line utility (`tools/itch_tool`, `ITCH_BUILD_TOOLS`): the + `stats`, `inspect`, `filter`, and `convert` subcommands, auto-detecting raw + ITCH versus `.pcap`/`.pcapng` input. + - pybind11 Python bindings (`python/`, `ITCH_BUILD_PYTHON`): a native module that + mirrors the pure-Python `itch` package's `MessageParser` and message-class + API (typed messages, `decode_price`) so it can serve as a faster drop-in + backend, plus the book engine and VWAP. Includes a `pyproject.toml` + (scikit-build-core) and pytest tests. + - Per-message Doxygen documentation for every ITCH message struct and field in + `itch/messages.hpp`, sourced from the message semantics in the `itch` package. + - **Phase 3 — Analytics & microstructure** ([FEATURES.md](FEATURES.md)). A header-only `itch::analytics` layer driven off the Phase 2 trade tape and book: - Bar builders (`analytics/bars.hpp`): OHLCV aggregation over time, tick, and diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b27e6f..c37a90a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,5 +39,11 @@ endif() if (${PROJECT_NAME}_BUILD_FUZZERS) add_subdirectory(fuzz) endif() +if (${PROJECT_NAME}_BUILD_TOOLS) + add_subdirectory(tools/itch_tool) +endif() +if (${PROJECT_NAME}_BUILD_PYTHON) + add_subdirectory(python) +endif() diff --git a/README.md b/README.md index 651c7d7..d6357c2 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,9 @@ The design of this ITCH parser is guided by three principles: - **Built-in Analytics**: Header-only microstructure layer — OHLCV bar builders (time/tick/volume clocks), VWAP/TWAP, spread, queue imbalance, order-flow imbalance, NOII surfacing, and auction reconstruction. See [Analytics](#analytics). +- **Interoperability**: CSV and Arrow/Parquet export, a batteries-included + `itch-tool` CLI (inspect/filter/stats/convert), and native Python bindings + (pybind11). See [Interoperability](#interoperability). - **High Throughput**: Multi-gigabyte-per-second parsing on modern hardware (see [Benchmarks](#benchmarks) for measured numbers). - **Allocation-Free Core**: The callback-based parsing loop performs zero dynamic memory allocations, minimizing latency and jitter. - **Type-Safe API**: All ITCH messages are deserialized into a `std::variant` of dedicated, packed `struct`s, ensuring compile-time safety. @@ -562,6 +565,37 @@ bars.flush(); double session_vwap = vwap.value(); ``` +### Interoperability + +Meet researchers in the tools they already use. + +**Output sinks.** `itch/io/csv_sink.hpp` flattens any message stream into a wide +CSV table (dependency-free). With `-DITCH_WITH_ARROW=ON` (the `arrow` vcpkg +feature), `itch/io/arrow_export.hpp` writes Parquet for pandas/Polars/DuckDB/Spark. + +**`itch-tool` CLI** (`-DITCH_BUILD_TOOLS=ON`). Inspect, filter, and convert feeds +without writing code; input may be a raw ITCH stream or a `.pcap`/`.pcapng` +capture (auto-detected): + +```bash +itch-tool stats data.itch # per-type message histogram +itch-tool inspect data.pcapng --limit 50 # human-readable dump +itch-tool filter data.itch --types AEP --out trades.csv +itch-tool convert data.itch --out data.csv # ITCH -> CSV (-> Parquet w/ Arrow) +``` + +**Python bindings** (`-DITCH_BUILD_PYTHON=ON`, pybind11). A native module that +mirrors the pure-Python [`itch`](https://github.com/bbalouki/itch) package's API +so it can act as a faster drop-in backend. See [python/README.md](python/README.md). + +```python +import itchcpp +parser = itchcpp.MessageParser() +for message in parser.parse_file("01302020.NASDAQ_ITCH50"): + if isinstance(message, itchcpp.AddOrderMessage): + print(message.stock, message.decode_price("price"), message.shares) +``` + --- ## API Reference: ITCH 5.0 Message Types diff --git a/VERSION.txt b/VERSION.txt index 88c5fb8..bc80560 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.4.0 +1.5.0 diff --git a/cmake/Helpers.cmake b/cmake/Helpers.cmake index d3b4def..77e2b63 100644 --- a/cmake/Helpers.cmake +++ b/cmake/Helpers.cmake @@ -9,6 +9,9 @@ option(${PROJECT_NAME}_BUILD_TESTS "Add tests" OFF) option(${PROJECT_NAME}_BUILD_BENCHMARKS "Add benchmark analysis" OFF) option(${PROJECT_NAME}_BUILD_EXAMPLES "Build some examples" OFF) option(${PROJECT_NAME}_BUILD_FUZZERS "Build libFuzzer targets (Clang only)" OFF) +option(${PROJECT_NAME}_BUILD_TOOLS "Build the itch-tool command-line utility" OFF) +option(${PROJECT_NAME}_BUILD_PYTHON "Build the pybind11 Python bindings" OFF) +option(${PROJECT_NAME}_WITH_ARROW "Enable Apache Arrow / Parquet export" OFF) set(${PROJECT_NAME}_PROJECT_ENV "DEV" CACHE STRING "Development environment") set_property(CACHE ${PROJECT_NAME}_PROJECT_ENV PROPERTY STRINGS "DEV" "PROD") diff --git a/include/itch/io/arrow_export.hpp b/include/itch/io/arrow_export.hpp new file mode 100644 index 0000000..363fdcc --- /dev/null +++ b/include/itch/io/arrow_export.hpp @@ -0,0 +1,62 @@ +#pragma once + +/// @file +/// @brief Apache Arrow / Parquet columnar export. +/// +/// This header is only meaningful when the library is built with +/// `-DITCH_WITH_ARROW=ON` (Arrow and Parquet provided via vcpkg). When the option +/// is off the class is not compiled, keeping the core dependency-free. The export +/// turns a feed into a columnar table that drops straight into pandas, Polars, +/// DuckDB, and Spark pipelines. + +#ifdef ITCH_WITH_ARROW + +#include +#include +#include + +#include "itch/messages.hpp" + +namespace arrow { +class Array; +} // namespace arrow + +namespace itch::io { + +/// @brief Accumulates parsed messages into Arrow columns and writes Parquet. +/// +/// Messages are flattened into the same normalized wide schema as `CsvSink`: +/// `message_type, timestamp, stock_locate, tracking_number, symbol,` +/// `reference_number, side, shares, price, match_number, printable, extra`. +/// Append each message with `append`, then call `write_parquet` to flush a file. +class ArrowExporter { + public: + ArrowExporter(); + ~ArrowExporter(); + ArrowExporter(const ArrowExporter&) = delete; + auto operator=(const ArrowExporter&) -> ArrowExporter& = delete; + ArrowExporter(ArrowExporter&&) noexcept = default; + auto operator=(ArrowExporter&&) noexcept -> ArrowExporter& = default; + + /// @brief Appends one message as a row to the column builders. + auto append(const Message& message) -> void; + + /// @brief The number of rows appended so far. + [[nodiscard]] auto rows() const noexcept -> std::uint64_t; + + /// @brief Finishes the builders and writes a Parquet file to `path`. + /// + /// @return True on success; on failure `error()` holds a description. + [[nodiscard]] auto write_parquet(const std::string& path) -> bool; + + /// @brief A description of the last failure, or empty on success. + [[nodiscard]] auto error() const -> const std::string&; + + private: + struct Impl; + std::unique_ptr m_impl; +}; + +} // namespace itch::io + +#endif // ITCH_WITH_ARROW diff --git a/include/itch/io/csv_sink.hpp b/include/itch/io/csv_sink.hpp new file mode 100644 index 0000000..09faca2 --- /dev/null +++ b/include/itch/io/csv_sink.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include +#include + +#include "itch/io/sink.hpp" +#include "itch/messages.hpp" + +namespace itch::io { + +/// @brief Writes parsed ITCH messages to a CSV stream as a flat, normalized table. +/// +/// ITCH messages are heterogeneous, so the sink flattens them into one wide row +/// schema sharing the common header plus the most frequently used per-type fields. +/// Fields a given message does not carry are left blank. This is the universal, +/// dependency-free output for quick inspection and legacy tooling; for columnar +/// analytics use the Arrow/Parquet exporter instead. +/// +/// Columns: `message_type,timestamp,stock_locate,tracking_number,symbol,` +/// `reference_number,side,shares,price,match_number,printable,extra`. +class CsvSink : public MessageSink { + public: + /// @brief Constructs a sink writing to `out`, emitting the header row unless + /// `write_header` is false (useful when appending to an existing file). + explicit CsvSink(std::ostream& out, bool write_header = true); + + /// @brief Writes one message as a CSV row. + auto write(const Message& message) -> void override; + + /// @brief Flushes the underlying stream. + auto flush() -> void override; + + /// @brief The number of rows written so far. + [[nodiscard]] auto rows_written() const noexcept -> std::uint64_t { return m_rows_written; } + + private: + std::ostream& m_out; + std::uint64_t m_rows_written {0}; +}; + +} // namespace itch::io diff --git a/include/itch/io/sink.hpp b/include/itch/io/sink.hpp new file mode 100644 index 0000000..5390ffd --- /dev/null +++ b/include/itch/io/sink.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include "itch/messages.hpp" + +namespace itch::io { + +/// @brief A generic streaming sink for parsed ITCH messages. +/// +/// Sinks are the universal output side of the library: a parse callback can hand +/// every message to a sink, which writes it somewhere (CSV, Arrow, a socket, a +/// counter). Implementations override `write`; `flush` is optional. +class MessageSink { + public: + MessageSink() = default; + MessageSink(const MessageSink&) = default; + MessageSink(MessageSink&&) noexcept = default; + auto operator=(const MessageSink&) -> MessageSink& = default; + auto operator=(MessageSink&&) noexcept -> MessageSink& = default; + virtual ~MessageSink() = default; + + /// @brief Consumes one parsed message. + virtual auto write(const Message& message) -> void = 0; + + /// @brief Flushes any buffered output. The default is a no-op. + virtual auto flush() -> void {} +}; + +} // namespace itch::io diff --git a/include/itch/messages.hpp b/include/itch/messages.hpp index 8050789..aa9285e 100644 --- a/include/itch/messages.hpp +++ b/include/itch/messages.hpp @@ -11,264 +11,415 @@ namespace itch { // DISABLE PADDING #pragma pack(push, 1) +/// @brief System Event (`S`): signals a market or data-feed handler event. +/// +/// Used to signal a market or data feed handler event. The `event_code` marks the +/// major points of the trading day: start and end of the messages stream, of +/// system hours, and of market hours. Consumers use it to bracket a session and +/// to know when the book is expected to be live. struct SystemEventMessage { - char message_type = 'S'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char event_code; + char message_type = 'S'; ///< Always 'S'. + uint16_t stock_locate; ///< Locate code; 0 for system-wide events. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char event_code; ///< 'O','S','Q','M','E','C' (see indicators::SYSTEM_EVENT_CODES). }; +/// @brief Stock Directory (`R`): the trading and listing reference data for a +/// security, disseminated at the start of each trading day. +/// +/// Disseminated for all active Nasdaq-traded symbols at the start of each trading +/// day. Market-data redistributors use it to populate fields such as the +/// Financial Status Indicator and Market Category so each security is classified +/// and displayed correctly. A security not named in any Stock Directory message +/// should not appear in later messages for that day. struct StockDirectoryMessage { - char message_type = 'R'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char market_category; - char financial_status_indicator; - uint32_t round_lot_size; - char round_lots_only; - char issue_classification; - char issue_sub_type[2]; - char authenticity; - char short_sale_threshold_indicator; - char ipo_flag; - char luld_ref; - char etp_flag; - uint32_t etp_leverage_factor; - char inverse_indicator; + char message_type = 'R'; ///< Always 'R'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char market_category; ///< Listing market (see indicators::MARKET_CATEGORY). + char financial_status_indicator; ///< Financial status of the issuer. + uint32_t round_lot_size; ///< Number of shares in a round lot. + char round_lots_only; ///< 'Y' if only round lots may be entered, else 'N'. + char issue_classification; ///< Security class (see indicators::ISSUE_CLASSIFICATION_VALUES). + char issue_sub_type[2]; ///< Security sub-type (see indicators::ISSUE_SUB_TYPE_VALUES). + char authenticity; ///< 'P' production / 'T' test security. + char short_sale_threshold_indicator; ///< Reg SHO threshold: 'Y','N',' '. + char ipo_flag; ///< 'Y' if a new IPO, 'N' if not, ' ' if not available. + char luld_ref; ///< LULD reference price tier. + char etp_flag; ///< 'Y' if an exchange-traded product, else 'N'. + uint32_t etp_leverage_factor; ///< Leverage factor of the ETP (if applicable). + char inverse_indicator; ///< 'Y' if the ETP is an inverse product. }; +/// @brief Stock Trading Action (`H`): a change in the trading status of a +/// security (halted, paused, quotation-only, or trading). +/// +/// An administrative message conveying the current trading status of a security. +/// It is sent before the open and again whenever the status changes, such as a +/// halt, a pause, a release for quotation, or a resumption of trading. The +/// `reason` code explains why the action occurred. struct StockTradingActionMessage { - char message_type = 'H'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char trading_state; - char reserved; - char reason[4]; + char message_type = 'H'; ///< Always 'H'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char trading_state; ///< 'H','P','Q','T' (see indicators::TRADING_STATES). + char reserved; ///< Reserved. + char reason[4]; ///< Trading-action reason code (see indicators::TRADING_ACTION_REASON_CODES). }; +/// @brief Reg SHO Short Sale Price Test Restriction (`Y`): the Reg SHO short-sale +/// restriction state for a security. +/// +/// Informs recipients of the SEC Rule 201 (Regulation SHO) short-sale price-test +/// restriction status for a security. It is sent both as a pre-open spin for every +/// security and intraday whenever the restriction status changes. struct RegSHOMessage { - char message_type = 'Y'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char reg_sho_action; + char message_type = 'Y'; ///< Always 'Y'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char reg_sho_action; ///< '0' no restriction, '1' restriction in effect, '2' remains. }; +/// @brief Market Participant Position (`L`): a market participant's (market +/// maker's) status in a security. +/// +/// Disseminated at the start of the trading day and whenever a status changes. It +/// provides, per firm registered in an issue, the Primary Market Maker status, the +/// Market Maker mode, and the Market Participant state. struct MarketParticipantPositionMessage { - char message_type = 'L'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char mpid[4]; - char stock[8]; - char primary_market_maker; - char market_maker_mode; - char market_participant_state; + char message_type = 'L'; ///< Always 'L'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char mpid[4]; ///< Market participant identifier. + char stock[8]; ///< Stock symbol, right padded with spaces. + char primary_market_maker; ///< 'Y' if the primary market maker, else 'N'. + char market_maker_mode; ///< Quotation mode (see indicators::MARKET_MAKER_MODE). + char market_participant_state; ///< State (see indicators::MARKET_PARTICIPANT_STATE). }; +/// @brief MWCB Decline Level (`V`): the Market-Wide Circuit Breaker breach points +/// for the day. +/// +/// Informs recipients what the daily Market-Wide Circuit Breaker breach points are +/// set to for the current trading day. The three levels correspond to the 5%, 13%, +/// and 20% S&P 500 decline thresholds that trigger a market-wide trading halt. +/// +/// @note Unlike every other price field (4 implied decimals), the three level +/// prices here carry 8 implied decimals; use `MwcbPrice` to interpret them. struct MWCBDeclineLevelMessage { - char message_type = 'V'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t level1; - uint64_t level2; - uint64_t level3; + char message_type = 'V'; ///< Always 'V'. + uint16_t stock_locate; ///< Locate code; 0 for this market-wide message. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t level1; ///< Level 1 (5%) breach value, 8 implied decimals. + uint64_t level2; ///< Level 2 (13%) breach value, 8 implied decimals. + uint64_t level3; ///< Level 3 (20%) breach value, 8 implied decimals. }; +/// @brief MWCB Status (`W`): notification that a Market-Wide Circuit Breaker level +/// has been breached. +/// +/// Informs recipients when a Market-Wide Circuit Breaker has breached one of the +/// established decline levels, which triggers the corresponding market-wide halt. struct MWCBStatusMessage { - char message_type = 'W'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char breached_level; + char message_type = 'W'; ///< Always 'W'. + uint16_t stock_locate; ///< Locate code; 0 for this market-wide message. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char breached_level; ///< '1', '2', or '3' for the level breached. }; +/// @brief IPO Quoting Period Update (`K`): the anticipated IPO quotation release +/// time and price for a security. +/// +/// Indicates the anticipated IPO quotation release time for a security. A +/// cancellation or postponement of the IPO is signalled by setting both the +/// release time and the price to zero. struct IPOQuotingPeriodUpdateMessage { - char message_type = 'K'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - uint32_t ipo_quotation_release_time; - char ipo_quotation_release_qualifier; - uint32_t ipo_price; + char message_type = 'K'; ///< Always 'K'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t ipo_quotation_release_time; ///< Seconds past midnight of the anticipated release. + char ipo_quotation_release_qualifier; ///< 'A' anticipated, 'C' cancelled/postponed. + uint32_t ipo_price; ///< IPO price (4 implied decimals). }; +/// @brief LULD Auction Collar (`J`): the auction collar thresholds for a security +/// in a Limit-Up Limit-Down trading pause. +/// +/// Indicates the auction collar thresholds within which a paused security may +/// reopen following a Limit-Up Limit-Down (LULD) trading pause. struct LULDAuctionCollarMessage { - char message_type = 'J'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - uint32_t auction_collar_reference_price; - uint32_t upper_auction_collar_price; - uint32_t lower_auction_collar_price; - uint32_t auction_collar_extension; + char message_type = 'J'; ///< Always 'J'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t auction_collar_reference_price; ///< Reference price for the collars (4 decimals). + uint32_t upper_auction_collar_price; ///< Upper auction collar price (4 decimals). + uint32_t lower_auction_collar_price; ///< Lower auction collar price (4 decimals). + uint32_t auction_collar_extension; ///< Number of collar extensions so far. }; +/// @brief Operational Halt (`h`): an operational halt or resumption for a security +/// on a specific market center. +/// +/// Indicates the operational status of a security: a service interruption that +/// affects only the designated market center. This differs from a Stock Trading +/// Action: an operational halt is a venue-level interruption rather than a +/// regulatory or volatility trading halt. struct OperationalHaltMessage { - char message_type = 'h'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char market_code; - char operational_halt_action; + char message_type = 'h'; ///< Always 'h'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char market_code; ///< 'Q' Nasdaq, 'B' BX, 'X' PSX. + char operational_halt_action; ///< 'H' halted, 'T' resumed. }; +/// @brief Add Order, No MPID Attribution (`A`): a new displayable order has been +/// accepted and placed on the book. +/// +/// Indicates that Nasdaq has accepted a new, unattributed order and added it to +/// the displayable book with a day-unique order reference number. That reference +/// number is used by every subsequent execute, cancel, delete, and replace +/// message that acts on the order. struct AddOrderMessage { - char message_type = 'A'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - char buy_sell_indicator; - uint32_t shares; - char stock[8]; - uint32_t price; + char message_type = 'A'; ///< Always 'A'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Day-unique reference number for the order. + char buy_sell_indicator; ///< 'B' buy, 'S' sell. + uint32_t shares; ///< Displayed share quantity. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t price; ///< Display price (4 implied decimals). }; +/// @brief Add Order, With MPID Attribution (`F`): like Add Order, but attributed +/// to a market participant. +/// +/// Indicates that Nasdaq has accepted a new attributed order or quotation and +/// added it to the displayable book. It is identical to an Add Order message +/// except that it also carries the attributing market participant identifier. struct AddOrderMPIDAttributionMessage { - char message_type = 'F'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - char buy_sell_indicator; - uint32_t shares; - char stock[8]; - uint32_t price; - char attribution[4]; + char message_type = 'F'; ///< Always 'F'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Day-unique reference number for the order. + char buy_sell_indicator; ///< 'B' buy, 'S' sell. + uint32_t shares; ///< Displayed share quantity. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t price; ///< Display price (4 implied decimals). + char attribution[4]; ///< Market participant identifier (MPID). }; +/// @brief Order Executed (`E`): a resting order was executed in whole or in part +/// at its display price. +/// +/// Sent when an order on the book is executed in whole or in part at its display +/// price. Several of these may be sent for the same order reference number; their +/// effects are cumulative, and the order is removed from the book once it is fully +/// executed. struct OrderExecutedMessage { - char message_type = 'E'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - uint32_t executed_shares; - uint64_t match_number; + char message_type = 'E'; ///< Always 'E'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the executed order. + uint32_t executed_shares; ///< Number of shares executed. + uint64_t match_number; ///< Day-unique match number for the execution. }; +/// @brief Order Executed With Price (`C`): a resting order was executed at a price +/// different from its display price (and may be non-printable). +/// +/// Sent when an order on the book executes at a price different from its initial +/// display price, so the execution price is carried explicitly. The execution may +/// be marked non-printable, in which case it is excluded from the time-and-sales +/// tape (typically to be printed later in aggregate). struct OrderExecutedWithPriceMessage { - char message_type = 'C'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - uint32_t executed_shares; - uint64_t match_number; - char printable; - uint32_t execution_price; + char message_type = 'C'; ///< Always 'C'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the executed order. + uint32_t executed_shares; ///< Number of shares executed. + uint64_t match_number; ///< Day-unique match number for the execution. + char printable; ///< 'Y' if the trade is printable to the tape, else 'N'. + uint32_t execution_price; ///< Price at which the order executed (4 decimals). }; +/// @brief Order Cancel (`X`): a partial cancellation reduced the shares of a +/// resting order. +/// +/// Sent when an order on the book is modified by a partial cancellation: the +/// specified number of shares is removed from the order's display size while the +/// order itself remains on the book. A full cancellation is conveyed by an Order +/// Delete (`D`) message instead. struct OrderCancelMessage { - char message_type = 'X'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - uint32_t cancelled_shares; + char message_type = 'X'; ///< Always 'X'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the cancelled order. + uint32_t cancelled_shares; ///< Number of shares cancelled. }; +/// @brief Order Delete (`D`): a resting order was cancelled in its entirety and +/// removed from the book. +/// +/// Sent when an order on the book is cancelled in full. All remaining shares +/// become inaccessible and the order must be removed from the book. struct OrderDeleteMessage { - char message_type = 'D'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; + char message_type = 'D'; ///< Always 'D'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the deleted order. }; +/// @brief Order Replace (`U`): a resting order was replaced with a new order at a +/// new reference number, size, and/or price (same side and security). +/// +/// Sent when an order on the book is cancel-replaced. The original order's shares +/// become inaccessible, and the replacement is assigned a new reference number +/// that is used for all subsequent updates. The side and security are unchanged; +/// only the price and size may differ. struct OrderReplaceMessage { - char message_type = 'U'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t original_order_reference_number; - uint64_t new_order_reference_number; - uint32_t shares; - uint32_t price; + char message_type = 'U'; ///< Always 'U'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t original_order_reference_number; ///< Reference number being replaced. + uint64_t new_order_reference_number; ///< New reference number for the order. + uint32_t shares; ///< New displayed share quantity. + uint32_t price; ///< New display price (4 implied decimals). }; +/// @brief Trade, Non-Cross (`P`): an execution of a non-displayable order. It does +/// not affect the visible book but is disseminated as a print. +/// +/// Provides execution details for normal match events involving non-displayable +/// order types, transmitted when such an order executes in whole or in part. +/// Because the order was never on the displayable book, this message does not +/// change book state; it exists so that consumers can include these executions in +/// the trade tape and volume calculations. struct NonCrossTradeMessage { - char message_type = 'P'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - char buy_sell_indicator; - uint32_t shares; - char stock[8]; - uint32_t price; - uint64_t match_number; + char message_type = 'P'; ///< Always 'P'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the non-displayed order. + char buy_sell_indicator; ///< 'B' buy, 'S' sell. + uint32_t shares; ///< Number of shares traded. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t price; ///< Trade price (4 implied decimals). + uint64_t match_number; ///< Day-unique match number for the trade. }; +/// @brief Cross Trade (`Q`): the result of a cross (opening, closing, halt/IPO +/// cross) for a security. +/// +/// Indicates that Nasdaq has completed the cross process for a security. It is +/// sent following the Opening Cross, the Closing Cross, and Extended Market Close +/// (EMC) cross events, and reports the matched volume and the single cross price. struct CrossTradeMessage { - char message_type = 'Q'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t shares; - char stock[8]; - uint32_t cross_price; - uint64_t match_number; - char cross_type; + char message_type = 'Q'; ///< Always 'Q'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t shares; ///< Number of shares matched in the cross. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t cross_price; ///< Price at which the cross executed (4 decimals). + uint64_t match_number; ///< Day-unique match number for the cross. + char cross_type; ///< 'O' open, 'C' close, 'H' halt/IPO, 'I' intraday. }; +/// @brief Broken Trade (`B`): a previously disseminated execution has been broken +/// and should be removed from any trade record. +/// +/// Sent whenever an execution on Nasdaq is broken under the clearly-erroneous +/// execution policy. A trade break is final and cannot be reinstated; consumers +/// should remove the referenced match number from their trade records. struct BrokenTradeMessage { - char message_type = 'B'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t match_number; + char message_type = 'B'; ///< Always 'B'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t match_number; ///< Match number of the execution being broken. }; +/// @brief Net Order Imbalance Indicator (`I`): the imbalance and price-discovery +/// data disseminated during the opening and closing crosses. +/// +/// Disseminates Net Order Imbalance Indicator data at regular intervals during the +/// cross sessions. It reports the paired and imbalanced share quantities and the +/// hypothetical clearing prices (far, near, and current reference price) the cross +/// would produce, so participants can react before the cross executes. struct NOIIMessage { - char message_type = 'I'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t paired_shares; - uint64_t imbalance_shares; - char imbalance_direction; - char stock[8]; - uint32_t far_price; - uint32_t near_price; - uint32_t current_reference_price; - char cross_type; - char price_variation_indicator; + char message_type = 'I'; ///< Always 'I'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t paired_shares; ///< Shares paired at the current reference price. + uint64_t imbalance_shares; ///< Shares not paired (the imbalance). + char imbalance_direction; ///< 'B' buy, 'S' sell, 'N' none, 'O' insufficient, 'P' paused. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t far_price; ///< Cross price using only eligible interest (4 decimals). + uint32_t near_price; ///< Cross price using all interest (4 decimals). + uint32_t current_reference_price; ///< Price the cross would occur at now (4 decimals). + char cross_type; ///< 'O' open, 'C' close, 'H' halt/IPO cross. + char price_variation_indicator; ///< Variation band (see indicators::PRICE_VARIATION_INDICATOR). }; +/// @brief Retail Price Improvement Indicator (`N`): the presence of retail price +/// improvement interest on the bid, the offer, or both. +/// +/// Identifies the presence of a retail price improvement interest indication (on +/// the bid, the offer, both, or none) for a Nasdaq-listed security. It signals +/// available retail liquidity without disclosing its size or price. struct RetailPriceImprovementIndicatorMessage { - char message_type = 'N'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char interest_flag; + char message_type = 'N'; ///< Always 'N'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char interest_flag; ///< 'B' bid, 'A' ask, 'C' both, 'N' none. }; +/// @brief Direct Listing with Capital Raise Price Discovery (`O`): price-discovery +/// data for a Direct Listing with a Capital Raise (DLCR) security. +/// +/// Disseminated only for Direct Listing with Capital Raise (DLCR) securities, once +/// the security passes its volatility test. It provides the allowable price +/// thresholds, the price range collars, and the anticipated execution price and +/// time used during the DLCR opening. struct DLCRMessage { - char message_type = 'O'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char open_eligibility_status; - uint32_t minimum_allowable_price; - uint32_t maximum_allowable_price; - uint32_t near_execution_price; - uint64_t near_execution_time; - uint32_t lower_price_range_collar; - uint32_t upper_price_range_collar; + char message_type = 'O'; ///< Always 'O'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char open_eligibility_status; ///< Whether the security is eligible to open. + uint32_t minimum_allowable_price; ///< Lowest allowable cross price (4 decimals). + uint32_t maximum_allowable_price; ///< Highest allowable cross price (4 decimals). + uint32_t near_execution_price; ///< Anticipated cross price (4 decimals). + uint64_t near_execution_time; ///< Time of the anticipated cross (ns past midnight). + uint32_t lower_price_range_collar; ///< Lower price range collar (4 decimals). + uint32_t upper_price_range_collar; ///< Upper price range collar (4 decimals). }; // RESTORE DEFAULT PADDING diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 0000000..c6d6d01 --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 3.25) + +# The Python bindings are built only when ITCH_BUILD_PYTHON is ON. pybind11 is +# resolved through vcpkg or a system install. +find_package(pybind11 CONFIG REQUIRED) + +pybind11_add_module(itchcpp_python src/bindings.cpp) +set_target_properties(itchcpp_python PROPERTIES OUTPUT_NAME "itchcpp") + +target_link_libraries(itchcpp_python PRIVATE itch::itch) + +# When built through scikit-build-core (pip install .) install into the wheel. +if(DEFINED SKBUILD) + install(TARGETS itchcpp_python LIBRARY DESTINATION .) +endif() diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..a128100 --- /dev/null +++ b/python/README.md @@ -0,0 +1,55 @@ +# itchcpp — Python bindings + +`itchcpp` is a native (C++) NASDAQ TotalView-ITCH 5.0 parser, order-book engine, +and analytics layer exposed to Python via pybind11. It is designed as a faster, +drop-in **backend for the pure-Python [`itch`](https://github.com/bbalouki/itch) +package**: it mirrors that package's `MessageParser` API and message-class field +and `decode_price` semantics, so existing code can switch to the native parser +with minimal changes. + +## Build / install + +The bindings require a C++20 compiler and pybind11. From the repository root: + +```bash +pip install ./python +``` + +This invokes scikit-build-core, which configures the parent CMake project with +`-DITCH_BUILD_PYTHON=ON` and builds the `itchcpp` extension module. + +To build just the extension with CMake directly: + +```bash +cmake -S . -B build -DITCH_BUILD_PYTHON=ON \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake +cmake --build build --target itchcpp_python +``` + +## Usage + +```python +import itchcpp + +parser = itchcpp.MessageParser() +for message in parser.parse_file("01302020.NASDAQ_ITCH50"): + if isinstance(message, itchcpp.AddOrderMessage): + print(message.stock, message.decode_price("price"), message.shares) + +# Full-market book reconstruction in one pass: +manager = itchcpp.BookManager() +with open("01302020.NASDAQ_ITCH50", "rb") as handle: + manager.process_stream(handle.read()) +book = manager.book_for_symbol("AAPL") +print(book.bbo().bid_price, book.bbo().ask_price) +``` + +## Relationship to the `itch` package + +The pure-Python `itch` package is already published on PyPI. Rather than rename or +break it, `itchcpp` ships as a separate, native package exposing the same API +surface (`MessageParser.parse_file`/`parse_stream`, typed message classes, +`decode_price`). The recommended migration is to have `itch` optionally import +`itchcpp` as its parsing backend when installed, falling back to the pure-Python +implementation otherwise — giving existing users a transparent speedup without a +breaking change. diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..a03b5a3 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,28 @@ +[build-system] +requires = ["scikit-build-core>=0.8", "pybind11>=2.11"] +build-backend = "scikit_build_core.build" + +[project] +name = "itchcpp" +version = "1.5.0" +description = "Native C++ NASDAQ TotalView-ITCH 5.0 parser, book engine, and analytics (a fast backend for the pure-Python `itch` package)." +readme = "README.md" +requires-python = ">=3.9" +license = { text = "MIT" } +authors = [{ name = "Bertin Balouki Simyeli" }] +keywords = ["nasdaq", "itch", "market-data", "order-book", "hft"] +classifiers = [ + "Programming Language :: C++", + "Programming Language :: Python :: 3", + "Topic :: Office/Business :: Financial :: Investment", + "License :: OSI Approved :: MIT License", +] + +[project.urls] +Homepage = "https://github.com/bbalouki/itchcpp" + +[tool.scikit-build] +# The bindings live under python/ but compile the library from the repo root. +cmake.source-dir = ".." +cmake.args = ["-DITCH_BUILD_PYTHON=ON"] +wheel.packages = [] diff --git a/python/src/bindings.cpp b/python/src/bindings.cpp new file mode 100644 index 0000000..0d10af1 --- /dev/null +++ b/python/src/bindings.cpp @@ -0,0 +1,392 @@ +#include +#include + +#include +#include +#include +#include + +#include "itch/analytics/vwap.hpp" +#include "itch/book/book_manager.hpp" +#include "itch/messages.hpp" +#include "itch/parser.hpp" + +namespace py = pybind11; + +namespace { + +// A thin parser facade mirroring the pure-Python `itch` package's MessageParser, +// so this native module can serve as a drop-in, faster backend for it. +struct MessageParser {}; + +// Standard price divisor (4 implied decimals); MWCB levels override this. +constexpr double STANDARD_DIVISOR = 10000.0; + +// Binds the fields common to every ITCH message plus the shared helpers. +template +auto bind_common(py::class_& cls, double price_divisor = STANDARD_DIVISOR) -> void { + cls.def_property_readonly("message_type", [](const MsgType& msg) { + return std::string(1, msg.message_type); + }); + cls.def_readonly("stock_locate", &MsgType::stock_locate); + cls.def_readonly("tracking_number", &MsgType::tracking_number); + cls.def_readonly("timestamp", &MsgType::timestamp); + // decode_price(name) divides the named raw integer field by the scale, exactly + // like the upstream package's MarketMessage.decode_price. + cls.def( + "decode_price", + [price_divisor](const py::object& self, const std::string& name) { + return self.attr(name.c_str()).cast() / price_divisor; + }, + py::arg("attribute_name") + ); +} + +// Exposes an 8-char stock symbol field as a trimmed Python string. +template +auto stock_property(py::class_& cls) -> void { + cls.def_property_readonly("stock", [](const MsgType& msg) { + return itch::to_string(msg.stock, itch::STOCK_LEN); + }); +} + +auto parse_buffer(const std::string& data) -> py::list { + py::list out; + itch::Parser parser; + parser.parse(data.data(), data.size(), [&out](const itch::Message& message) { + std::visit([&out](const auto& concrete) { out.append(py::cast(concrete)); }, message); + }); + return out; +} + +} // namespace + +PYBIND11_MODULE(itchcpp, mod) { + mod.doc() = + "Native (C++) NASDAQ TotalView-ITCH 5.0 parser. A faster drop-in backend " + "for the pure-Python `itch` package: it exposes a MessageParser plus typed " + "message classes with the same field and decode_price semantics."; + + // --- Message classes ----------------------------------------------------- + { + py::class_ cls {mod, "SystemEventMessage"}; + bind_common(cls); + cls.def_property_readonly("event_code", [](const itch::SystemEventMessage& m) { + return std::string(1, m.event_code); + }); + } + { + py::class_ cls {mod, "StockDirectoryMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_readonly("round_lot_size", &itch::StockDirectoryMessage::round_lot_size); + cls.def_property_readonly("market_category", [](const itch::StockDirectoryMessage& m) { + return std::string(1, m.market_category); + }); + } + { + py::class_ cls {mod, "StockTradingActionMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_property_readonly("trading_state", [](const itch::StockTradingActionMessage& m) { + return std::string(1, m.trading_state); + }); + cls.def_property_readonly("reason", [](const itch::StockTradingActionMessage& m) { + return itch::to_string(m.reason, 4); + }); + } + { + py::class_ cls {mod, "RegSHOMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_property_readonly("reg_sho_action", [](const itch::RegSHOMessage& m) { + return std::string(1, m.reg_sho_action); + }); + } + { + py::class_ cls { + mod, "MarketParticipantPositionMessage" + }; + bind_common(cls); + stock_property(cls); + cls.def_property_readonly("mpid", [](const itch::MarketParticipantPositionMessage& m) { + return itch::to_string(m.mpid, 4); + }); + } + { + py::class_ cls {mod, "MWCBDeclineLevelMessage"}; + bind_common(cls, 1.0E8); // MWCB levels carry 8 implied decimals. + cls.def_readonly("level1", &itch::MWCBDeclineLevelMessage::level1); + cls.def_readonly("level2", &itch::MWCBDeclineLevelMessage::level2); + cls.def_readonly("level3", &itch::MWCBDeclineLevelMessage::level3); + } + { + py::class_ cls {mod, "MWCBStatusMessage"}; + bind_common(cls); + cls.def_property_readonly("breached_level", [](const itch::MWCBStatusMessage& m) { + return std::string(1, m.breached_level); + }); + } + { + py::class_ cls {mod, "IPOQuotingPeriodUpdateMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_readonly("ipo_price", &itch::IPOQuotingPeriodUpdateMessage::ipo_price); + cls.def_readonly( + "ipo_quotation_release_time", + &itch::IPOQuotingPeriodUpdateMessage::ipo_quotation_release_time + ); + } + { + py::class_ cls {mod, "LULDAuctionCollarMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_readonly( + "auction_collar_reference_price", + &itch::LULDAuctionCollarMessage::auction_collar_reference_price + ); + cls.def_readonly( + "upper_auction_collar_price", + &itch::LULDAuctionCollarMessage::upper_auction_collar_price + ); + cls.def_readonly( + "lower_auction_collar_price", + &itch::LULDAuctionCollarMessage::lower_auction_collar_price + ); + } + { + py::class_ cls {mod, "OperationalHaltMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_property_readonly("market_code", [](const itch::OperationalHaltMessage& m) { + return std::string(1, m.market_code); + }); + cls.def_property_readonly( + "operational_halt_action", [](const itch::OperationalHaltMessage& m) { + return std::string(1, m.operational_halt_action); + } + ); + } + { + py::class_ cls {mod, "AddOrderMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_readonly("order_reference_number", &itch::AddOrderMessage::order_reference_number); + cls.def_readonly("shares", &itch::AddOrderMessage::shares); + cls.def_readonly("price", &itch::AddOrderMessage::price); + cls.def_property_readonly("buy_sell_indicator", [](const itch::AddOrderMessage& m) { + return std::string(1, m.buy_sell_indicator); + }); + } + { + py::class_ cls { + mod, "AddOrderMPIDAttributionMessage" + }; + bind_common(cls); + stock_property(cls); + cls.def_readonly( + "order_reference_number", &itch::AddOrderMPIDAttributionMessage::order_reference_number + ); + cls.def_readonly("shares", &itch::AddOrderMPIDAttributionMessage::shares); + cls.def_readonly("price", &itch::AddOrderMPIDAttributionMessage::price); + cls.def_property_readonly( + "buy_sell_indicator", [](const itch::AddOrderMPIDAttributionMessage& m) { + return std::string(1, m.buy_sell_indicator); + } + ); + cls.def_property_readonly("attribution", [](const itch::AddOrderMPIDAttributionMessage& m) { + return itch::to_string(m.attribution, 4); + }); + } + { + py::class_ cls {mod, "OrderExecutedMessage"}; + bind_common(cls); + cls.def_readonly( + "order_reference_number", &itch::OrderExecutedMessage::order_reference_number + ); + cls.def_readonly("executed_shares", &itch::OrderExecutedMessage::executed_shares); + cls.def_readonly("match_number", &itch::OrderExecutedMessage::match_number); + } + { + py::class_ cls {mod, "OrderExecutedWithPriceMessage"}; + bind_common(cls); + cls.def_readonly( + "order_reference_number", &itch::OrderExecutedWithPriceMessage::order_reference_number + ); + cls.def_readonly("executed_shares", &itch::OrderExecutedWithPriceMessage::executed_shares); + cls.def_readonly("match_number", &itch::OrderExecutedWithPriceMessage::match_number); + cls.def_readonly("execution_price", &itch::OrderExecutedWithPriceMessage::execution_price); + cls.def_property_readonly("printable", [](const itch::OrderExecutedWithPriceMessage& m) { + return std::string(1, m.printable); + }); + } + { + py::class_ cls {mod, "OrderCancelMessage"}; + bind_common(cls); + cls.def_readonly( + "order_reference_number", &itch::OrderCancelMessage::order_reference_number + ); + cls.def_readonly("cancelled_shares", &itch::OrderCancelMessage::cancelled_shares); + } + { + py::class_ cls {mod, "OrderDeleteMessage"}; + bind_common(cls); + cls.def_readonly( + "order_reference_number", &itch::OrderDeleteMessage::order_reference_number + ); + } + { + py::class_ cls {mod, "OrderReplaceMessage"}; + bind_common(cls); + cls.def_readonly( + "original_order_reference_number", + &itch::OrderReplaceMessage::original_order_reference_number + ); + cls.def_readonly( + "new_order_reference_number", &itch::OrderReplaceMessage::new_order_reference_number + ); + cls.def_readonly("shares", &itch::OrderReplaceMessage::shares); + cls.def_readonly("price", &itch::OrderReplaceMessage::price); + } + { + py::class_ cls {mod, "NonCrossTradeMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_readonly( + "order_reference_number", &itch::NonCrossTradeMessage::order_reference_number + ); + cls.def_readonly("shares", &itch::NonCrossTradeMessage::shares); + cls.def_readonly("price", &itch::NonCrossTradeMessage::price); + cls.def_readonly("match_number", &itch::NonCrossTradeMessage::match_number); + cls.def_property_readonly("buy_sell_indicator", [](const itch::NonCrossTradeMessage& m) { + return std::string(1, m.buy_sell_indicator); + }); + } + { + py::class_ cls {mod, "CrossTradeMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_readonly("shares", &itch::CrossTradeMessage::shares); + cls.def_readonly("cross_price", &itch::CrossTradeMessage::cross_price); + cls.def_readonly("match_number", &itch::CrossTradeMessage::match_number); + cls.def_property_readonly("cross_type", [](const itch::CrossTradeMessage& m) { + return std::string(1, m.cross_type); + }); + } + { + py::class_ cls {mod, "BrokenTradeMessage"}; + bind_common(cls); + cls.def_readonly("match_number", &itch::BrokenTradeMessage::match_number); + } + { + py::class_ cls {mod, "NOIIMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_readonly("paired_shares", &itch::NOIIMessage::paired_shares); + cls.def_readonly("imbalance_shares", &itch::NOIIMessage::imbalance_shares); + cls.def_readonly("far_price", &itch::NOIIMessage::far_price); + cls.def_readonly("near_price", &itch::NOIIMessage::near_price); + cls.def_readonly("current_reference_price", &itch::NOIIMessage::current_reference_price); + cls.def_property_readonly("imbalance_direction", [](const itch::NOIIMessage& m) { + return std::string(1, m.imbalance_direction); + }); + cls.def_property_readonly("cross_type", [](const itch::NOIIMessage& m) { + return std::string(1, m.cross_type); + }); + } + { + py::class_ cls { + mod, "RetailPriceImprovementIndicatorMessage" + }; + bind_common(cls); + stock_property(cls); + cls.def_property_readonly( + "interest_flag", [](const itch::RetailPriceImprovementIndicatorMessage& m) { + return std::string(1, m.interest_flag); + } + ); + } + { + py::class_ cls {mod, "DLCRMessage"}; + bind_common(cls); + stock_property(cls); + cls.def_readonly("near_execution_price", &itch::DLCRMessage::near_execution_price); + cls.def_readonly("minimum_allowable_price", &itch::DLCRMessage::minimum_allowable_price); + cls.def_readonly("maximum_allowable_price", &itch::DLCRMessage::maximum_allowable_price); + } + + // --- Parser facade ------------------------------------------------------- + py::class_(mod, "MessageParser") + .def(py::init<>()) + .def( + "parse_stream", + [](MessageParser&, const py::bytes& data) { return parse_buffer(std::string {data}); }, + py::arg("raw_binary_data"), + "Parse messages from raw bytes, returning a list of message objects." + ) + .def( + "parse_file", + [](MessageParser&, const std::string& path) { + std::ifstream file {path, std::ios::binary}; + const std::string data {std::istreambuf_iterator {file}, {}}; + return parse_buffer(data); + }, + py::arg("itch_file"), + "Read and parse all messages from a binary ITCH file path." + ); + + // --- Book engine --------------------------------------------------------- + py::class_(mod, "Bbo") + .def_readonly("has_bid", &itch::book::Bbo::has_bid) + .def_readonly("has_ask", &itch::book::Bbo::has_ask) + .def_readonly("bid_shares", &itch::book::Bbo::bid_shares) + .def_readonly("ask_shares", &itch::book::Bbo::ask_shares) + .def_property_readonly( + "bid_price", [](const itch::book::Bbo& b) { return b.bid_price.to_double(); } + ) + .def_property_readonly("ask_price", [](const itch::book::Bbo& b) { + return b.ask_price.to_double(); + }); + + py::class_(mod, "L3Book") + .def_property_readonly("symbol", &itch::book::L3Book::symbol) + .def("bbo", &itch::book::L3Book::bbo); + + py::class_(mod, "BookManager") + .def(py::init<>()) + .def("track_symbol", &itch::book::BookManager::track_symbol, py::arg("symbol")) + .def( + "process_stream", + [](itch::book::BookManager& manager, const py::bytes& data) { + const std::string buffer {data}; + itch::Parser parser; + parser.parse(buffer.data(), buffer.size(), [&manager](const itch::Message& msg) { + manager.process(msg); + }); + }, + py::arg("raw_binary_data"), + "Parse a raw ITCH buffer and update every symbol's book in one pass." + ) + .def("book_count", &itch::book::BookManager::book_count) + .def( + "book_for_symbol", + &itch::book::BookManager::book_for_symbol, + py::arg("symbol"), + py::return_value_policy::reference_internal + ); + + // --- Analytics ----------------------------------------------------------- + py::class_(mod, "Vwap") + .def(py::init<>()) + .def( + "add", + [](itch::analytics::Vwap& vwap, std::uint32_t raw_price, std::uint64_t shares) { + vwap.add(itch::StandardPrice {raw_price}, shares); + }, + py::arg("raw_price"), + py::arg("shares") + ) + .def("value", &itch::analytics::Vwap::value) + .def("volume", &itch::analytics::Vwap::volume) + .def("reset", &itch::analytics::Vwap::reset); +} diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py new file mode 100644 index 0000000..6377fb1 --- /dev/null +++ b/python/tests/test_bindings.py @@ -0,0 +1,67 @@ +"""Tests for the native ``itchcpp`` bindings. + +These mirror the pure-Python ``itch`` package's API so the native module can be a +drop-in backend: a ``MessageParser`` producing typed message objects with +``decode_price`` semantics, plus the book engine and analytics. +""" + +import struct + +import itchcpp + + +def _frame(payload: bytes) -> bytes: + return struct.pack(">H", len(payload)) + payload + + +def _add_order(locate: int, ref: int, side: bytes, shares: int, sym: bytes, price: int) -> bytes: + payload = ( + b"A" + + struct.pack(">HH", locate, 0) + + (1234).to_bytes(6, "big") + + struct.pack(">Q", ref) + + side + + struct.pack(">I", shares) + + sym.ljust(8) + + struct.pack(">I", price) + ) + return _frame(payload) + + +def test_parse_stream_yields_typed_messages() -> None: + buffer = _add_order(7, 42, b"B", 500, b"AAPL", 1_500_000) + parser = itchcpp.MessageParser() + messages = parser.parse_stream(buffer) + + assert len(messages) == 1 + message = messages[0] + assert isinstance(message, itchcpp.AddOrderMessage) + assert message.message_type == "A" + assert message.stock == "AAPL" + assert message.shares == 500 + assert message.order_reference_number == 42 + # decode_price applies the 4-decimal scale, matching the upstream API. + assert message.decode_price("price") == 150.0 + + +def test_book_manager_reconstructs_bbo() -> None: + buffer = _add_order(7, 1, b"B", 300, b"AAPL", 1_500_000) + _add_order( + 7, 2, b"S", 100, b"AAPL", 1_500_200 + ) + manager = itchcpp.BookManager() + manager.process_stream(buffer) + + assert manager.book_count() == 1 + book = manager.book_for_symbol("AAPL") + bbo = book.bbo() + assert bbo.bid_price == 150.0 + assert bbo.ask_price == 150.02 + assert bbo.bid_shares == 300 + + +def test_vwap_accumulates() -> None: + vwap = itchcpp.Vwap() + vwap.add(100, 10) + vwap.add(200, 30) + assert vwap.volume() == 40 + assert abs(vwap.value() - (175.0 / 10000.0)) < 1e-12 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 712ab1c..a9c0daa 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,7 +13,20 @@ add_library( transport/pcap.cpp book/l3_book.cpp book/book_manager.cpp + io/csv_sink.cpp + io/arrow_export.cpp ) + +# Apache Arrow / Parquet columnar export is optional and off by default so the +# core library stays dependency-free. The arrow_export.cpp translation unit is +# empty unless ITCH_WITH_ARROW is defined. +if(${PROJECT_NAME}_WITH_ARROW) + find_package(Arrow CONFIG REQUIRED) + find_package(Parquet CONFIG REQUIRED) + target_compile_definitions(itch PUBLIC ITCH_WITH_ARROW) + target_link_libraries(itch PUBLIC Arrow::arrow_shared Parquet::parquet_shared) + message(STATUS "ITCH: Apache Arrow / Parquet export enabled") +endif() add_library(itch::itch ALIAS itch) if (NOT ANDROID) target_link_libraries( diff --git a/src/io/arrow_export.cpp b/src/io/arrow_export.cpp new file mode 100644 index 0000000..1cbb033 --- /dev/null +++ b/src/io/arrow_export.cpp @@ -0,0 +1,257 @@ +#include "itch/io/arrow_export.hpp" + +#ifdef ITCH_WITH_ARROW + +#include +#include +#include + +#include +#include + +#include "itch/price.hpp" + +namespace itch::io { + +struct ArrowExporter::Impl { + arrow::StringBuilder message_type; + arrow::UInt64Builder timestamp; + arrow::UInt16Builder stock_locate; + arrow::UInt16Builder tracking_number; + arrow::StringBuilder symbol; + arrow::UInt64Builder reference_number; + arrow::StringBuilder side; + arrow::UInt64Builder shares; + arrow::DoubleBuilder price; + arrow::UInt64Builder match_number; + arrow::StringBuilder printable; + arrow::StringBuilder extra; + std::uint64_t row_count {0}; + std::string last_error; + + // Builds the common header columns shared by every message type. + template + auto append_common(const MsgType& msg) -> void { + (void)message_type.Append(std::string {msg.message_type}); + (void)timestamp.Append(msg.timestamp); + (void)stock_locate.Append(msg.stock_locate); + (void)tracking_number.Append(msg.tracking_number); + } + + // Appends nulls/empties for the type-specific columns, overwritten as needed. + auto append_blank_specific() -> void { + (void)symbol.AppendEmptyValue(); + (void)reference_number.AppendNull(); + (void)side.AppendEmptyValue(); + (void)shares.AppendNull(); + (void)price.AppendNull(); + (void)match_number.AppendNull(); + (void)printable.AppendEmptyValue(); + (void)extra.AppendEmptyValue(); + } +}; + +ArrowExporter::ArrowExporter() : m_impl {std::make_unique()} {} +ArrowExporter::~ArrowExporter() = default; + +auto ArrowExporter::rows() const noexcept -> std::uint64_t { return m_impl->row_count; } +auto ArrowExporter::error() const -> const std::string& { return m_impl->last_error; } + +auto ArrowExporter::append(const Message& message) -> void { + Impl& impl = *m_impl; + std::visit([&impl](const auto& msg) { impl.append_common(msg); }, message); + + // Default every type-specific column to null/empty, then set what applies. + // Replacing the convenience of AppendEmptyValue with explicit per-type code + // keeps the column lengths in lockstep with the header columns. + std::string symbol_value; + bool has_reference = false; + std::uint64_t reference = 0; + std::string side_value; + bool has_shares = false; + std::uint64_t shares_value = 0; + bool has_price = false; + double price_value = 0.0; + bool has_match = false; + std::uint64_t match_value = 0; + std::string printable_value; + std::string extra_value; + + if (const auto* add = std::get_if(&message)) { + symbol_value = to_string(add->stock, STOCK_LEN); + has_reference = true; + reference = add->order_reference_number; + side_value = std::string {add->buy_sell_indicator}; + has_shares = true; + shares_value = add->shares; + has_price = true; + price_value = StandardPrice {add->price}.to_double(); + } else if (const auto* add_mpid = std::get_if(&message)) { + symbol_value = to_string(add_mpid->stock, STOCK_LEN); + has_reference = true; + reference = add_mpid->order_reference_number; + side_value = std::string {add_mpid->buy_sell_indicator}; + has_shares = true; + shares_value = add_mpid->shares; + has_price = true; + price_value = StandardPrice {add_mpid->price}.to_double(); + extra_value = "mpid=" + to_string(add_mpid->attribution, 4); + } else if (const auto* exec = std::get_if(&message)) { + has_reference = true; + reference = exec->order_reference_number; + has_shares = true; + shares_value = exec->executed_shares; + has_match = true; + match_value = exec->match_number; + } else if (const auto* exec_px = std::get_if(&message)) { + has_reference = true; + reference = exec_px->order_reference_number; + has_shares = true; + shares_value = exec_px->executed_shares; + has_price = true; + price_value = StandardPrice {exec_px->execution_price}.to_double(); + has_match = true; + match_value = exec_px->match_number; + printable_value = std::string {exec_px->printable}; + } else if (const auto* cancel = std::get_if(&message)) { + has_reference = true; + reference = cancel->order_reference_number; + has_shares = true; + shares_value = cancel->cancelled_shares; + } else if (const auto* del = std::get_if(&message)) { + has_reference = true; + reference = del->order_reference_number; + } else if (const auto* replace = std::get_if(&message)) { + has_reference = true; + reference = replace->new_order_reference_number; + has_shares = true; + shares_value = replace->shares; + has_price = true; + price_value = StandardPrice {replace->price}.to_double(); + extra_value = "orig=" + std::to_string(replace->original_order_reference_number); + } else if (const auto* trade = std::get_if(&message)) { + symbol_value = to_string(trade->stock, STOCK_LEN); + has_reference = true; + reference = trade->order_reference_number; + side_value = std::string {trade->buy_sell_indicator}; + has_shares = true; + shares_value = trade->shares; + has_price = true; + price_value = StandardPrice {trade->price}.to_double(); + has_match = true; + match_value = trade->match_number; + } else if (const auto* cross = std::get_if(&message)) { + symbol_value = to_string(cross->stock, STOCK_LEN); + has_shares = true; + shares_value = cross->shares; + has_price = true; + price_value = StandardPrice {cross->cross_price}.to_double(); + has_match = true; + match_value = cross->match_number; + extra_value = std::string {"cross="} + cross->cross_type; + } else if (const auto* event = std::get_if(&message)) { + extra_value = std::string {"event="} + event->event_code; + } else if (const auto* directory = std::get_if(&message)) { + symbol_value = to_string(directory->stock, STOCK_LEN); + } + + (void)impl.symbol.Append(symbol_value); + if (has_reference) { + (void)impl.reference_number.Append(reference); + } else { + (void)impl.reference_number.AppendNull(); + } + (void)impl.side.Append(side_value); + if (has_shares) { + (void)impl.shares.Append(shares_value); + } else { + (void)impl.shares.AppendNull(); + } + if (has_price) { + (void)impl.price.Append(price_value); + } else { + (void)impl.price.AppendNull(); + } + if (has_match) { + (void)impl.match_number.Append(match_value); + } else { + (void)impl.match_number.AppendNull(); + } + (void)impl.printable.Append(printable_value); + (void)impl.extra.Append(extra_value); + ++impl.row_count; +} + +auto ArrowExporter::write_parquet(const std::string& path) -> bool { + Impl& impl = *m_impl; + + auto finish = + [&impl](arrow::ArrayBuilder& builder, std::shared_ptr& out) -> bool { + const arrow::Status status = builder.Finish(&out); + if (!status.ok()) { + impl.last_error = status.ToString(); + return false; + } + return true; + }; + + std::shared_ptr columns[12]; + arrow::ArrayBuilder* builders[12] = { + &impl.message_type, + &impl.timestamp, + &impl.stock_locate, + &impl.tracking_number, + &impl.symbol, + &impl.reference_number, + &impl.side, + &impl.shares, + &impl.price, + &impl.match_number, + &impl.printable, + &impl.extra + }; + for (int index = 0; index < 12; ++index) { + if (!finish(*builders[index], columns[index])) { + return false; + } + } + + auto schema = arrow::schema({ + arrow::field("message_type", arrow::utf8()), + arrow::field("timestamp", arrow::uint64()), + arrow::field("stock_locate", arrow::uint16()), + arrow::field("tracking_number", arrow::uint16()), + arrow::field("symbol", arrow::utf8()), + arrow::field("reference_number", arrow::uint64()), + arrow::field("side", arrow::utf8()), + arrow::field("shares", arrow::uint64()), + arrow::field("price", arrow::float64()), + arrow::field("match_number", arrow::uint64()), + arrow::field("printable", arrow::utf8()), + arrow::field("extra", arrow::utf8()), + }); + + std::vector> column_vector {columns, columns + 12}; + const auto table = arrow::Table::Make(schema, column_vector); + + auto outfile_result = arrow::io::FileOutputStream::Open(path); + if (!outfile_result.ok()) { + impl.last_error = outfile_result.status().ToString(); + return false; + } + const auto status = parquet::arrow::WriteTable( + *table, + arrow::default_memory_pool(), + *outfile_result, + impl.row_count > 0 ? impl.row_count : 1 + ); + if (!status.ok()) { + impl.last_error = status.ToString(); + return false; + } + return true; +} + +} // namespace itch::io + +#endif // ITCH_WITH_ARROW diff --git a/src/io/csv_sink.cpp b/src/io/csv_sink.cpp new file mode 100644 index 0000000..c3c483a --- /dev/null +++ b/src/io/csv_sink.cpp @@ -0,0 +1,141 @@ +#include "itch/io/csv_sink.hpp" + +#include +#include +#include + +#include "itch/price.hpp" + +namespace itch::io { + +namespace { + +/// @brief The flattened CSV view of one message, with blanks for absent fields. +struct Row { + char message_type {'\0'}; + std::uint64_t timestamp {0}; + std::uint16_t stock_locate {0}; + std::uint16_t tracking_number {0}; + std::string symbol; + std::string reference_number; + char side {'\0'}; + std::string shares; + std::string price; + std::string match_number; + char printable {'\0'}; + std::string extra; +}; + +// Builds the common header fields shared by every message. +template +auto fill_common(Row& row, const MsgType& msg) -> void { + row.message_type = msg.message_type; + row.timestamp = msg.timestamp; + row.stock_locate = msg.stock_locate; + row.tracking_number = msg.tracking_number; +} + +// Quotes a value only if it contains a comma; symbols never do, but be safe. +auto field(const std::string& value) -> std::string { + if (value.find(',') != std::string::npos) { + return std::format("\"{}\"", value); + } + return value; +} + +auto build_row(const Message& message) -> Row { + Row row {}; + std::visit([&row](const auto& msg) { fill_common(row, msg); }, message); + + if (const auto* add = std::get_if(&message)) { + row.symbol = to_string(add->stock, STOCK_LEN); + row.reference_number = std::to_string(add->order_reference_number); + row.side = add->buy_sell_indicator; + row.shares = std::to_string(add->shares); + row.price = StandardPrice {add->price}.to_string(); + } else if (const auto* add_mpid = std::get_if(&message)) { + row.symbol = to_string(add_mpid->stock, STOCK_LEN); + row.reference_number = std::to_string(add_mpid->order_reference_number); + row.side = add_mpid->buy_sell_indicator; + row.shares = std::to_string(add_mpid->shares); + row.price = StandardPrice {add_mpid->price}.to_string(); + row.extra = std::format("mpid={}", to_string(add_mpid->attribution, 4)); + } else if (const auto* exec = std::get_if(&message)) { + row.reference_number = std::to_string(exec->order_reference_number); + row.shares = std::to_string(exec->executed_shares); + row.match_number = std::to_string(exec->match_number); + } else if (const auto* exec_px = std::get_if(&message)) { + row.reference_number = std::to_string(exec_px->order_reference_number); + row.shares = std::to_string(exec_px->executed_shares); + row.price = StandardPrice {exec_px->execution_price}.to_string(); + row.match_number = std::to_string(exec_px->match_number); + row.printable = exec_px->printable; + } else if (const auto* cancel = std::get_if(&message)) { + row.reference_number = std::to_string(cancel->order_reference_number); + row.shares = std::to_string(cancel->cancelled_shares); + } else if (const auto* del = std::get_if(&message)) { + row.reference_number = std::to_string(del->order_reference_number); + } else if (const auto* replace = std::get_if(&message)) { + row.reference_number = std::to_string(replace->new_order_reference_number); + row.shares = std::to_string(replace->shares); + row.price = StandardPrice {replace->price}.to_string(); + row.extra = std::format("orig={}", replace->original_order_reference_number); + } else if (const auto* trade = std::get_if(&message)) { + row.symbol = to_string(trade->stock, STOCK_LEN); + row.reference_number = std::to_string(trade->order_reference_number); + row.side = trade->buy_sell_indicator; + row.shares = std::to_string(trade->shares); + row.price = StandardPrice {trade->price}.to_string(); + row.match_number = std::to_string(trade->match_number); + } else if (const auto* cross = std::get_if(&message)) { + row.symbol = to_string(cross->stock, STOCK_LEN); + row.shares = std::to_string(cross->shares); + row.price = StandardPrice {cross->cross_price}.to_string(); + row.match_number = std::to_string(cross->match_number); + row.extra = std::format("cross={}", cross->cross_type); + } else if (const auto* event = std::get_if(&message)) { + row.extra = std::format("event={}", event->event_code); + } else if (const auto* directory = std::get_if(&message)) { + row.symbol = to_string(directory->stock, STOCK_LEN); + row.extra = std::format("category={}", directory->market_category); + } + return row; +} + +// Renders a single character field, blank when unset. +auto char_field(char value) -> std::string { + return value == '\0' ? std::string {} : std::string {value}; +} + +} // namespace + +CsvSink::CsvSink(std::ostream& out, bool write_header) : m_out {out} { + if (write_header) { + m_out << "message_type,timestamp,stock_locate,tracking_number,symbol,reference_number," + "side,shares,price,match_number,printable,extra\n"; + } +} + +auto CsvSink::write(const Message& message) -> void { + const Row row = build_row(message); + m_out << std::format( + "{},{},{},{},{},{},{},{},{},{},{},{}\n", + row.message_type, + row.timestamp, + row.stock_locate, + row.tracking_number, + field(row.symbol), + row.reference_number, + char_field(row.side), + row.shares, + row.price, + row.match_number, + char_field(row.printable), + field(row.extra) + ); + ++m_rows_written; +} + +auto CsvSink::flush() -> void { m_out.flush(); } + +} // namespace itch::io diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 6281f70..ba5ea4d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable( book/test_book.cpp book/test_overlay.cpp analytics/test_analytics.cpp + io/test_csv_sink.cpp ) target_include_directories(itch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/tests/io/test_csv_sink.cpp b/tests/io/test_csv_sink.cpp new file mode 100644 index 0000000..d7d51e3 --- /dev/null +++ b/tests/io/test_csv_sink.cpp @@ -0,0 +1,46 @@ +#include + +#include +#include +#include +#include + +#include "itch/io/csv_sink.hpp" +#include "itch/parser.hpp" +#include "transport/frame_builders.hpp" + +namespace { + +auto build_buffer(const std::vector>& payloads) -> std::vector { + std::vector buffer; + for (const auto& payload : payloads) { + const auto frame = itch::test::length_prefixed(payload); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + return buffer; +} + +} // namespace + +TEST(CsvSink, WritesHeaderAndNormalizedRows) { + const auto buffer = build_buffer({ + itch::test::system_event_payload(1000, 'O'), + itch::test::add_order_payload(7, 42, 'B', 500, "AAPL", 1500000), + }); + + std::ostringstream out; + itch::io::CsvSink sink {out}; + itch::Parser parser; + parser.parse(std::span {buffer}, [&](const itch::Message& msg) { + sink.write(msg); + }); + sink.flush(); + + const std::string text = out.str(); + EXPECT_NE(text.find("message_type,timestamp,stock_locate"), std::string::npos); + // System event row carries its event code in the extra column. + EXPECT_NE(text.find("event=O"), std::string::npos); + // Add order row carries symbol, side, shares, and a 4-decimal price. + EXPECT_NE(text.find("AAPL,42,B,500,150.0000"), std::string::npos); + EXPECT_EQ(sink.rows_written(), 2U); +} diff --git a/tools/itch_tool/CMakeLists.txt b/tools/itch_tool/CMakeLists.txt new file mode 100644 index 0000000..a3d9a85 --- /dev/null +++ b/tools/itch_tool/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.25) + +include(GNUInstallDirs) + +add_executable(itch_tool main.cpp) +set_target_properties(itch_tool PROPERTIES OUTPUT_NAME "itch-tool") + +target_link_libraries( + itch_tool PRIVATE + itch::itch + warnings::strict +) + +install(TARGETS itch_tool RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/tools/itch_tool/main.cpp b/tools/itch_tool/main.cpp new file mode 100644 index 0000000..889422a --- /dev/null +++ b/tools/itch_tool/main.cpp @@ -0,0 +1,174 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/io/csv_sink.hpp" +#include "itch/parser.hpp" +#include "itch/transport/pcap.hpp" + +namespace { + +// Reads an entire file into a byte buffer. +auto read_file(const std::string& path) -> std::vector { + std::ifstream file {path, std::ios::binary}; + if (!file) { + return {}; + } + std::vector raw {std::istreambuf_iterator {file}, {}}; + std::vector bytes(raw.size()); + if (!raw.empty()) { + std::memcpy(bytes.data(), raw.data(), raw.size()); + } + return bytes; +} + +// Drives a callback over every message in a file, auto-detecting whether the +// input is a pcap/pcapng capture or a raw length-prefixed ITCH stream. +auto for_each_message(std::span data, const itch::MessageCallback& callback) + -> void { + itch::transport::PcapReader reader {callback}; + if (reader.read(data)) { + return; // The input was a capture file. + } + itch::Parser parser; + parser.parse(data, callback); +} + +// Parses a comma-or-concatenated list of message-type characters (e.g. "AEP"). +auto parse_type_filter(std::string_view spec) -> std::array { + std::array wanted {}; + for (char chr : spec) { + if (chr != ',') { + wanted[static_cast(chr)] = true; + } + } + return wanted; +} + +auto message_type_of(const itch::Message& message) -> char { + return std::visit([](const auto& msg) { return msg.message_type; }, message); +} + +auto cmd_stats(std::span data) -> int { + std::array counts {}; + std::uint64_t total = 0; + for_each_message(data, [&](const itch::Message& msg) { + ++counts[static_cast(message_type_of(msg))]; + ++total; + }); + std::println("{:<6} {:>14}", "type", "count"); + for (std::size_t type = 0; type < counts.size(); ++type) { + if (counts[type] > 0) { + std::println("{:<6} {:>14}", static_cast(type), counts[type]); + } + } + std::println("{:<6} {:>14}", "TOTAL", total); + return 0; +} + +auto cmd_inspect(std::span data, std::uint64_t limit) -> int { + std::uint64_t shown = 0; + for_each_message(data, [&](const itch::Message& msg) { + if (shown < limit) { + std::cout << msg << '\n'; // Message provides operator<<. + ++shown; + } + }); + std::println("(showed {} messages)", shown); + return 0; +} + +auto cmd_filter_or_convert( + std::span data, + const std::array& wanted, + bool has_filter, + const std::string& out_path +) -> int { + std::ofstream file_out; + if (!out_path.empty()) { + file_out.open(out_path, std::ios::binary); + if (!file_out) { + std::println(stderr, "Error: cannot open output '{}'.", out_path); + return 1; + } + } + std::ostream& out = out_path.empty() ? std::cout : file_out; + itch::io::CsvSink sink {out}; + for_each_message(data, [&](const itch::Message& msg) { + if (!has_filter || wanted[static_cast(message_type_of(msg))]) { + sink.write(msg); + } + }); + sink.flush(); + std::println(stderr, "Wrote {} rows.", sink.rows_written()); + return 0; +} + +auto usage(const char* program) -> int { + std::println(stderr, "itch-tool - inspect, filter, and convert NASDAQ ITCH 5.0 data"); + std::println(stderr, "Usage:"); + std::println(stderr, " {} stats ", program); + std::println(stderr, " {} inspect [--limit N]", program); + std::println(stderr, " {} filter --types [--out ]", program); + std::println(stderr, " {} convert [--to csv] [--out ]", program); + std::println(stderr, "Input may be a raw ITCH stream or a .pcap/.pcapng capture."); + return 1; +} + +} // namespace + +auto main(int argc, char* argv[]) -> int { + const std::vector args {argv, argv + argc}; + if (args.size() < 3) { + return usage(argv[0]); + } + const std::string& command = args[1]; + const std::string& path = args[2]; + + std::string out_path; + std::string types; + std::uint64_t limit = 20; + std::array wanted {}; + bool has_filter = false; + for (std::size_t index = 3; index < args.size(); ++index) { + if (args[index] == "--out" && index + 1 < args.size()) { + out_path = args[++index]; + } else if (args[index] == "--types" && index + 1 < args.size()) { + types = args[++index]; + wanted = parse_type_filter(types); + has_filter = true; + } else if (args[index] == "--limit" && index + 1 < args.size()) { + limit = std::stoull(args[++index]); + } else if (args[index] == "--to" && index + 1 < args.size()) { + ++index; // Only csv is supported without the Arrow build option. + } + } + + const std::vector data = read_file(path); + if (data.empty()) { + std::println(stderr, "Error: cannot read '{}' (missing or empty).", path); + return 1; + } + const std::span view {data}; + + if (command == "stats") { + return cmd_stats(view); + } + if (command == "inspect") { + return cmd_inspect(view, limit); + } + if (command == "filter") { + return cmd_filter_or_convert(view, wanted, has_filter, out_path); + } + if (command == "convert") { + return cmd_filter_or_convert(view, wanted, has_filter, out_path); + } + return usage(argv[0]); +} diff --git a/vcpkg.json b/vcpkg.json index 165ef1b..f268b88 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -3,5 +3,24 @@ "gtest", "benchmark" ], + "features": { + "python": { + "description": "Build the pybind11 Python bindings", + "dependencies": [ + "pybind11" + ] + }, + "arrow": { + "description": "Enable Apache Arrow / Parquet columnar export", + "dependencies": [ + { + "name": "arrow", + "features": [ + "parquet" + ] + } + ] + } + }, "builtin-baseline": "5bf0c55239da398b8c6f450818c9e28d36bf9966" } From 35f81467893c1008687b808ad2827b635eebe326 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Mon, 29 Jun 2026 01:34:49 +0100 Subject: [PATCH 030/144] Add Phase 5 simulation & ecosystem Implement the FEATURES.md Phase 5 layer, closing the roadmap: - replay.hpp/replay.cpp: ReplayEngine drives a callback paced by message timestamps at original or scaled wall-clock speed for backtesting. - encoder.hpp/encoder.cpp: encode_message/encode_frame serialize any Message back to valid wire bytes (inverse of the parser), guaranteeing parse(encode(msg)) == msg; also backs the Python to_bytes(). - venue.hpp: a VenuePolicy concept and the concrete Nasdaq50 policy as the extension seam for additional venues/versions (BX/PSX, ITCH 4.1), without new decoders (extension points only). - Packaging: conanfile.py, CONTRIBUTING.md, and a documented versioning/ABI compatibility policy. Adds encoder round-trip, replay pacing, and venue-policy tests, plus README and CHANGELOG entries. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 15 +++ CONTRIBUTING.md | 74 ++++++++++++ README.md | 35 ++++++ VERSION.txt | 2 +- conanfile.py | 69 +++++++++++ include/itch/encoder.hpp | 28 +++++ include/itch/replay.hpp | 46 ++++++++ include/itch/venue.hpp | 45 +++++++ python/src/bindings.cpp | 8 ++ src/CMakeLists.txt | 2 + src/encoder.cpp | 248 +++++++++++++++++++++++++++++++++++++++ src/replay.cpp | 52 ++++++++ tests/CMakeLists.txt | 1 + tests/test_encoder.cpp | 125 ++++++++++++++++++++ 14 files changed, 749 insertions(+), 1 deletion(-) create mode 100644 CONTRIBUTING.md create mode 100644 conanfile.py create mode 100644 include/itch/encoder.hpp create mode 100644 include/itch/replay.hpp create mode 100644 include/itch/venue.hpp create mode 100644 src/encoder.cpp create mode 100644 src/replay.cpp create mode 100644 tests/test_encoder.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 7830155..3dbafeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Phase 5 — Simulation & ecosystem** ([FEATURES.md](FEATURES.md)): + - Replay engine (`itch/replay.hpp`): `ReplayEngine` drives a callback paced by + the messages' nanosecond timestamps at original or scaled wall-clock speed, for + realistic backtesting and system simulation. + - ITCH encoder / writer (`itch/encoder.hpp`): `encode_message` and `encode_frame` + serialize any `Message` back to valid wire bytes (the inverse of the parser), + guaranteeing `parse(encode(msg)) == msg`. This synthesizes streams for tests, + scenario generation, and golden fixtures, and backs the Python `to_bytes()`. + - Multi-venue / multi-version extension seam (`itch/venue.hpp`): a `VenuePolicy` + concept and the concrete `Nasdaq50` policy, so additional ITCH-like venues and + versions (BX/PSX, ITCH 4.1) can be added without rewriting the dispatch + machinery. Only the NASDAQ 5.0 policy is implemented. + - Packaging maturity: a Conan recipe (`conanfile.py`), a `CONTRIBUTING.md`, and a + documented versioning/ABI compatibility policy. + - **Phase 4 — Research & interoperability** ([FEATURES.md](FEATURES.md)): - CSV and streaming sinks (`itch/io/sink.hpp`, `itch/io/csv_sink.hpp`): a generic `MessageSink` interface and a `CsvSink` that flattens every message type into a diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e04d074 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,74 @@ +# Contributing to ITCHCPP + +Thanks for your interest in improving ITCHCPP. This guide covers how to build, +test, and submit changes, and the compatibility policy the project follows. + +## Getting started + +ITCHCPP targets **C++20** (and builds under C++23). You need a recent compiler +(GCC 12+, Clang 15+, or MSVC 19.3x) and CMake 3.25+. Developer dependencies +(GoogleTest, Google Benchmark, and the optional pybind11/Arrow features) are +resolved through vcpkg. + +```bash +cmake -S . -B build \ + -DITCH_BUILD_TESTS=ON -DITCH_BUILD_EXAMPLES=ON -DITCH_BUILD_TOOLS=ON \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake +cmake --build build +ctest --test-dir build --output-on-failure +``` + +Optional features are off by default and enabled with: + +- `-DITCH_BUILD_BENCHMARKS=ON` — Google Benchmark suites. +- `-DITCH_BUILD_FUZZERS=ON` — libFuzzer targets (Clang only). +- `-DITCH_BUILD_TOOLS=ON` — the `itch-tool` CLI. +- `-DITCH_BUILD_PYTHON=ON` — the pybind11 Python bindings (needs the `python` + vcpkg feature: `-DVCPKG_MANIFEST_FEATURES=python`). +- `-DITCH_WITH_ARROW=ON` — Apache Arrow / Parquet export (needs the `arrow` + vcpkg feature: `-DVCPKG_MANIFEST_FEATURES=arrow`). + +## Code style + +- Follow the rules in [CLAUDE.md](CLAUDE.md): `snake_case` functions/variables, + `PascalCase` types, `m_` private members, `SCREAMING_SNAKE_CASE` constants, + symbols at least three characters, trailing return types, brace initialization, + and `std::print`/`std::format` for output. +- Run `clang-format` (the repo ships a `.clang-format`) and `clang-tidy` before + submitting; CI enforces both. +- Document the _why_ in comments; use Doxygen `///` for public API. + +## Tests + +- Add tests under `tests/` mirroring the source layout, using GoogleTest. +- Cover edge and error cases, not just the happy path. Parsers and decoders that + consume untrusted input should also be exercised by a fuzz target. +- Keep tests deterministic and fast. + +## Pull requests + +- CI must pass: the build/test matrix (GCC, Clang, MSVC; C++20 and C++23), + clang-format, clang-tidy, sanitizers (ASAN/UBSAN), coverage, fuzzing, and the + Python-binding job. +- Keep the public API surface minimal; mark deprecations with + `[[deprecated("reason")]]` and provide a migration path. +- Update [CHANGELOG.md](CHANGELOG.md) under `[Unreleased]` and follow the Boy + Scout Rule. + +## Versioning and compatibility policy + +ITCHCPP follows [Semantic Versioning 2.0.0](https://semver.org). A breaking change +is any backward-incompatible modification to the public API, which for this project +includes: + +- the C++ headers under `include/itch/` (types, function signatures, struct + layouts of the wire message structs), +- the message wire formats and the encoder/parser round-trip contract, +- the `itch-tool` command-line interface, +- the Python module's public API. + +The build options and CMake target names (`itch::itch`) are also part of the +public surface. Within a major version, source compatibility is preserved; the +binary (ABI) is **not** guaranteed stable across minor versions, so link against a +single version. New venues/protocols are added behind the `itch::venue::VenuePolicy` +seam without breaking existing policies. diff --git a/README.md b/README.md index d6357c2..d19b3e0 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,9 @@ The design of this ITCH parser is guided by three principles: - **Interoperability**: CSV and Arrow/Parquet export, a batteries-included `itch-tool` CLI (inspect/filter/stats/convert), and native Python bindings (pybind11). See [Interoperability](#interoperability). +- **Simulation & Ecosystem**: Timestamp-paced replay engine, a full ITCH + encoder/writer (`parse(encode(msg)) == msg`), multi-venue extension seams, and + Conan/vcpkg packaging. See [Simulation](#simulation--ecosystem). - **High Throughput**: Multi-gigabyte-per-second parsing on modern hardware (see [Benchmarks](#benchmarks) for measured numbers). - **Allocation-Free Core**: The callback-based parsing loop performs zero dynamic memory allocations, minimizing latency and jitter. - **Type-Safe API**: All ITCH messages are deserialized into a `std::variant` of dedicated, packed `struct`s, ensuring compile-time safety. @@ -596,6 +599,38 @@ for message in parser.parse_file("01302020.NASDAQ_ITCH50"): print(message.stock, message.decode_price("price"), message.shares) ``` +### Simulation & Ecosystem + +**Replay engine** (`itch/replay.hpp`). Drive a consumer at the feed's original +cadence (or a scaled speed) for realistic backtesting: + +```cpp +#include "itch/replay.hpp" + +itch::ReplayEngine engine{10.0}; // 10x real time; <= 0 means as fast as possible +engine.replay(buffer, [](const itch::Message& msg) { /* paced by timestamps */ }); +``` + +**Encoder / writer** (`itch/encoder.hpp`). Serialize any message back to valid +wire bytes, with a guaranteed `parse(encode(msg)) == msg` round-trip — used to +synthesize streams and golden fixtures: + +```cpp +#include "itch/encoder.hpp" + +std::vector frame = itch::encode_frame(message); // length-prefixed +``` + +**Multi-venue seam** (`itch/venue.hpp`). NASDAQ TotalView-ITCH 5.0 is the only +implemented venue; `itch::venue::VenuePolicy` (modelled by `itch::venue::Nasdaq50`) +is the extension point for adding BX/PSX or older ITCH versions without rewriting +the dispatch machinery. + +**Packaging.** The library is consumable through vcpkg (manifest with optional +`python` and `arrow` features) and Conan (`conanfile.py`). See +[CONTRIBUTING.md](CONTRIBUTING.md) for the build options and the versioning/ABI +compatibility policy. + --- ## API Reference: ITCH 5.0 Message Types diff --git a/VERSION.txt b/VERSION.txt index bc80560..dc1e644 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -1.5.0 +1.6.0 diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000..84b1578 --- /dev/null +++ b/conanfile.py @@ -0,0 +1,69 @@ +"""Conan recipe for ITCHCPP. + +Packages the header-only-friendly ITCH 5.0 library (parser, transport decoders, +book engine, analytics, encoder, replay) so downstream projects can depend on it +through Conan. Tests, benchmarks, tools, and the optional Arrow/Python features are +not part of the consumed package; enable them from a source build instead. +""" + +import os + +from conan import ConanFile +from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.files import copy, load + + +class ItchConan(ConanFile): + name = "itchcpp" + license = "MIT" + url = "https://github.com/bbalouki/itchcpp" + description = ( + "High-performance C++20 NASDAQ TotalView-ITCH 5.0 parser, order-book " + "engine, analytics, and feed-ingestion library." + ) + topics = ("nasdaq", "itch", "market-data", "order-book", "hft", "finance") + settings = "os", "compiler", "build_type", "arch" + options = {"with_arrow": [True, False]} + default_options = {"with_arrow": False} + exports_sources = ( + "CMakeLists.txt", + "VERSION.txt", + "cmake/*", + "src/*", + "include/*", + ) + + def set_version(self): + self.version = load(self, os.path.join(self.recipe_folder, "VERSION.txt")).strip() + + def requirements(self): + if self.options.with_arrow: + self.requires("arrow/15.0.0") + + def layout(self): + cmake_layout(self) + + def generate(self): + toolchain = CMakeToolchain(self) + toolchain.variables["ITCH_WITH_ARROW"] = self.options.with_arrow + toolchain.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + copy( + self, + "LICENSE", + src=self.source_folder, + dst=os.path.join(self.package_folder, "licenses"), + ) + + def package_info(self): + self.cpp_info.libs = ["itch"] + self.cpp_info.set_property("cmake_file_name", "itch") + self.cpp_info.set_property("cmake_target_name", "itch::itch") diff --git a/include/itch/encoder.hpp b/include/itch/encoder.hpp new file mode 100644 index 0000000..5ff5d9b --- /dev/null +++ b/include/itch/encoder.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include +#include + +#include "itch/messages.hpp" + +namespace itch { + +/// @brief Serializes parsed ITCH messages back to valid wire bytes. +/// +/// The encoder is the inverse of the parser: it writes each message's fields in +/// the spec's order and network (big-endian) byte order, including the 48-bit +/// timestamp. It is used to synthesize valid ITCH streams for testing, scenario +/// generation, and golden fixtures, and it guarantees the round-trip property +/// `parse(encode(msg)) == msg` for every supported message type. + +/// @brief Encodes a message body and header without the 2-byte length prefix. +/// +/// @return The raw message bytes (type byte first), exactly as they appear inside +/// a frame. +[[nodiscard]] auto encode_message(const Message& message) -> std::vector; + +/// @brief Encodes a complete length-prefixed frame (2-byte big-endian length +/// followed by the message bytes), ready to concatenate into a stream. +[[nodiscard]] auto encode_frame(const Message& message) -> std::vector; + +} // namespace itch diff --git a/include/itch/replay.hpp b/include/itch/replay.hpp new file mode 100644 index 0000000..5009b54 --- /dev/null +++ b/include/itch/replay.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include +#include + +#include "itch/parser.hpp" + +namespace itch { + +/// @brief Replays a parsed ITCH stream paced by message timestamps. +/// +/// For realistic backtesting and system simulation, a consumer often needs the +/// feed delivered at its original wall-clock cadence rather than as fast as the +/// CPU can parse it. The replay engine parses a buffer and invokes the callback +/// for each message, sleeping between messages so the inter-message gaps match the +/// differences in their nanoseconds-past-midnight timestamps, optionally scaled by +/// a speed factor. +class ReplayEngine { + public: + /// @brief Constructs an engine with a speed multiplier. + /// + /// @param speed_multiplier Wall-clock speed relative to the original feed: 1.0 + /// replays in real time, 2.0 twice as fast, 0.5 half speed. A value + /// less than or equal to 0 replays with no pacing (as fast as possible). + explicit ReplayEngine(double speed_multiplier = 1.0) noexcept + : m_speed_multiplier {speed_multiplier} {} + + /// @brief Sets the speed multiplier (see the constructor). + auto set_speed(double speed_multiplier) noexcept -> void { + m_speed_multiplier = speed_multiplier; + } + + /// @brief The current speed multiplier. + [[nodiscard]] auto speed() const noexcept -> double { return m_speed_multiplier; } + + /// @brief Parses `data` and invokes `callback` for each message, paced by the + /// message timestamps. + /// + /// @return The number of messages replayed. + auto replay(std::span data, const MessageCallback& callback) -> std::uint64_t; + + private: + double m_speed_multiplier {1.0}; +}; + +} // namespace itch diff --git a/include/itch/venue.hpp b/include/itch/venue.hpp new file mode 100644 index 0000000..97bd556 --- /dev/null +++ b/include/itch/venue.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include + +#include "itch/detail/wire.hpp" + +namespace itch::venue { + +/// @brief Concept describing a venue/protocol policy. +/// +/// ITCHCPP today implements NASDAQ TotalView-ITCH 5.0 concretely. This concept is +/// the extension seam for adding ITCH-like venues and versions (NASDAQ BX/PSX, +/// older ITCH 4.1, and other venue feeds) without rewriting the dispatch +/// machinery: a policy names itself and enumerates its message-type registry, and +/// the dispatch-table builder can be parameterized on it. +/// +/// A conforming policy provides: +/// - `static constexpr std::string_view name()` identifying the venue/version. +/// - `static void for_each_message_type(Visitor&&)` invoking +/// `visitor.template operator()(char)` once per message type, the same +/// shape as `itch::detail::for_each_message_type`. +template +concept VenuePolicy = requires { + { Policy::name() } -> std::convertible_to; +}; + +/// @brief The concrete policy for NASDAQ TotalView-ITCH 5.0 (the only venue +/// implemented today). New venues are added by writing another policy of +/// the same shape and parameterizing the dispatch builder on it. +struct Nasdaq50 { + [[nodiscard]] static constexpr auto name() noexcept -> std::string_view { + return "NASDAQ TotalView-ITCH 5.0"; + } + + /// @brief Enumerates this venue's message-type registry. + template + static constexpr auto for_each_message_type(Visitor&& visitor) -> void { + detail::for_each_message_type(static_cast(visitor)); + } +}; + +static_assert(VenuePolicy); + +} // namespace itch::venue diff --git a/python/src/bindings.cpp b/python/src/bindings.cpp index 0d10af1..58535be 100644 --- a/python/src/bindings.cpp +++ b/python/src/bindings.cpp @@ -8,6 +8,7 @@ #include "itch/analytics/vwap.hpp" #include "itch/book/book_manager.hpp" +#include "itch/encoder.hpp" #include "itch/messages.hpp" #include "itch/parser.hpp" @@ -40,6 +41,13 @@ auto bind_common(py::class_& cls, double price_divisor = STANDARD_DIVIS }, py::arg("attribute_name") ); + // to_bytes() serializes the message back to wire form, matching the upstream + // MarketMessage.to_bytes (here it returns the message body without the frame + // length prefix). + cls.def("to_bytes", [](const MsgType& msg) { + const std::vector bytes = itch::encode_message(itch::Message {msg}); + return py::bytes(reinterpret_cast(bytes.data()), bytes.size()); // NOLINT + }); } // Exposes an 8-char stock symbol field as a trimmed Python string. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a9c0daa..3312848 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,6 +15,8 @@ add_library( book/book_manager.cpp io/csv_sink.cpp io/arrow_export.cpp + encoder.cpp + replay.cpp ) # Apache Arrow / Parquet columnar export is optional and off by default so the diff --git a/src/encoder.cpp b/src/encoder.cpp new file mode 100644 index 0000000..97383d4 --- /dev/null +++ b/src/encoder.cpp @@ -0,0 +1,248 @@ +#include "itch/encoder.hpp" + +#include +#include +#include +#include + +#include "itch/parser.hpp" // utils::from_big_endian + +namespace itch { + +namespace { + +/// @brief Appends fields to a byte buffer in network (big-endian) order. +class ByteWriter { + public: + // Appends an integral field of width sizeof(IntType) in big-endian order. + template + auto put(IntType value) -> void { + const auto big = utils::from_big_endian(value); + std::array bytes {}; + std::memcpy(bytes.data(), &big, sizeof(IntType)); + m_bytes.insert(m_bytes.end(), bytes.begin(), bytes.end()); + } + + auto put_char(char value) -> void { m_bytes.push_back(static_cast(value)); } + + // Appends a fixed-width character field exactly as stored (no trimming). + auto put_chars(const char* source, std::size_t size) -> void { + for (std::size_t index = 0; index < size; ++index) { + m_bytes.push_back(static_cast(source[index])); + } + } + + // Appends the 48-bit ITCH timestamp (2-byte high, 4-byte low), big-endian. + auto put_timestamp(std::uint64_t timestamp) -> void { + constexpr int LOWER_SHIFT = 32; + put(static_cast(timestamp >> LOWER_SHIFT)); + put(static_cast(timestamp & 0xFFFFFFFFULL)); + } + + [[nodiscard]] auto take() -> std::vector { return std::move(m_bytes); } + + private: + std::vector m_bytes; +}; + +// Writes the header common to every message: type, locate, tracking, timestamp. +template +auto write_header(ByteWriter& writer, const MsgType& msg) -> void { + writer.put_char(msg.message_type); + writer.put(msg.stock_locate); + writer.put(msg.tracking_number); + writer.put_timestamp(msg.timestamp); +} + +auto write_body(ByteWriter& writer, const SystemEventMessage& msg) -> void { + writer.put_char(msg.event_code); +} + +auto write_body(ByteWriter& writer, const StockDirectoryMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.market_category); + writer.put_char(msg.financial_status_indicator); + writer.put(msg.round_lot_size); + writer.put_char(msg.round_lots_only); + writer.put_char(msg.issue_classification); + writer.put_chars(msg.issue_sub_type, 2); + writer.put_char(msg.authenticity); + writer.put_char(msg.short_sale_threshold_indicator); + writer.put_char(msg.ipo_flag); + writer.put_char(msg.luld_ref); + writer.put_char(msg.etp_flag); + writer.put(msg.etp_leverage_factor); + writer.put_char(msg.inverse_indicator); +} + +auto write_body(ByteWriter& writer, const StockTradingActionMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.trading_state); + writer.put_char(msg.reserved); + writer.put_chars(msg.reason, 4); +} + +auto write_body(ByteWriter& writer, const RegSHOMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.reg_sho_action); +} + +auto write_body(ByteWriter& writer, const MarketParticipantPositionMessage& msg) -> void { + writer.put_chars(msg.mpid, 4); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.primary_market_maker); + writer.put_char(msg.market_maker_mode); + writer.put_char(msg.market_participant_state); +} + +auto write_body(ByteWriter& writer, const MWCBDeclineLevelMessage& msg) -> void { + writer.put(msg.level1); + writer.put(msg.level2); + writer.put(msg.level3); +} + +auto write_body(ByteWriter& writer, const MWCBStatusMessage& msg) -> void { + writer.put_char(msg.breached_level); +} + +auto write_body(ByteWriter& writer, const IPOQuotingPeriodUpdateMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.ipo_quotation_release_time); + writer.put_char(msg.ipo_quotation_release_qualifier); + writer.put(msg.ipo_price); +} + +auto write_body(ByteWriter& writer, const LULDAuctionCollarMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.auction_collar_reference_price); + writer.put(msg.upper_auction_collar_price); + writer.put(msg.lower_auction_collar_price); + writer.put(msg.auction_collar_extension); +} + +auto write_body(ByteWriter& writer, const OperationalHaltMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.market_code); + writer.put_char(msg.operational_halt_action); +} + +auto write_body(ByteWriter& writer, const AddOrderMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put_char(msg.buy_sell_indicator); + writer.put(msg.shares); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.price); +} + +auto write_body(ByteWriter& writer, const AddOrderMPIDAttributionMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put_char(msg.buy_sell_indicator); + writer.put(msg.shares); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.price); + writer.put_chars(msg.attribution, 4); +} + +auto write_body(ByteWriter& writer, const OrderExecutedMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put(msg.executed_shares); + writer.put(msg.match_number); +} + +auto write_body(ByteWriter& writer, const OrderExecutedWithPriceMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put(msg.executed_shares); + writer.put(msg.match_number); + writer.put_char(msg.printable); + writer.put(msg.execution_price); +} + +auto write_body(ByteWriter& writer, const OrderCancelMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put(msg.cancelled_shares); +} + +auto write_body(ByteWriter& writer, const OrderDeleteMessage& msg) -> void { + writer.put(msg.order_reference_number); +} + +auto write_body(ByteWriter& writer, const OrderReplaceMessage& msg) -> void { + writer.put(msg.original_order_reference_number); + writer.put(msg.new_order_reference_number); + writer.put(msg.shares); + writer.put(msg.price); +} + +auto write_body(ByteWriter& writer, const NonCrossTradeMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put_char(msg.buy_sell_indicator); + writer.put(msg.shares); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.price); + writer.put(msg.match_number); +} + +auto write_body(ByteWriter& writer, const CrossTradeMessage& msg) -> void { + writer.put(msg.shares); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.cross_price); + writer.put(msg.match_number); + writer.put_char(msg.cross_type); +} + +auto write_body(ByteWriter& writer, const BrokenTradeMessage& msg) -> void { + writer.put(msg.match_number); +} + +auto write_body(ByteWriter& writer, const NOIIMessage& msg) -> void { + writer.put(msg.paired_shares); + writer.put(msg.imbalance_shares); + writer.put_char(msg.imbalance_direction); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.far_price); + writer.put(msg.near_price); + writer.put(msg.current_reference_price); + writer.put_char(msg.cross_type); + writer.put_char(msg.price_variation_indicator); +} + +auto write_body(ByteWriter& writer, const RetailPriceImprovementIndicatorMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.interest_flag); +} + +auto write_body(ByteWriter& writer, const DLCRMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.open_eligibility_status); + writer.put(msg.minimum_allowable_price); + writer.put(msg.maximum_allowable_price); + writer.put(msg.near_execution_price); + writer.put(msg.near_execution_time); + writer.put(msg.lower_price_range_collar); + writer.put(msg.upper_price_range_collar); +} + +} // namespace + +auto encode_message(const Message& message) -> std::vector { + ByteWriter writer; + std::visit( + [&writer](const auto& concrete) { + write_header(writer, concrete); + write_body(writer, concrete); + }, + message + ); + return writer.take(); +} + +auto encode_frame(const Message& message) -> std::vector { + const std::vector body = encode_message(message); + ByteWriter writer; + writer.put(static_cast(body.size())); + std::vector frame = writer.take(); + frame.insert(frame.end(), body.begin(), body.end()); + return frame; +} + +} // namespace itch diff --git a/src/replay.cpp b/src/replay.cpp new file mode 100644 index 0000000..67dec3f --- /dev/null +++ b/src/replay.cpp @@ -0,0 +1,52 @@ +#include "itch/replay.hpp" + +#include +#include +#include + +namespace itch { + +namespace { + +// Extracts the nanoseconds-past-midnight timestamp from any message. +auto timestamp_of(const Message& message) -> std::uint64_t { + return std::visit([](const auto& concrete) { return concrete.timestamp; }, message); +} + +} // namespace + +auto ReplayEngine::replay(std::span data, const MessageCallback& callback) + -> std::uint64_t { + using clock = std::chrono::steady_clock; + const bool paced = m_speed_multiplier > 0.0; + + std::uint64_t replayed = 0; + bool have_base = false; + std::uint64_t base_timestamp = 0; + clock::time_point base_wall {}; + + Parser parser; + parser.parse(data, [&](const Message& message) { + if (paced) { + const std::uint64_t feed_timestamp = timestamp_of(message); + if (!have_base) { + base_timestamp = feed_timestamp; + base_wall = clock::now(); + have_base = true; + } else if (feed_timestamp > base_timestamp) { + // Scale the elapsed feed time by the speed multiplier and sleep + // until that point relative to the first message's wall time. + const double elapsed_ns = + static_cast(feed_timestamp - base_timestamp) / m_speed_multiplier; + const auto target = + base_wall + std::chrono::nanoseconds {static_cast(elapsed_ns)}; + std::this_thread::sleep_until(target); + } + } + callback(message); + ++replayed; + }); + return replayed; +} + +} // namespace itch diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index ba5ea4d..fb4a925 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -27,6 +27,7 @@ add_executable( book/test_overlay.cpp analytics/test_analytics.cpp io/test_csv_sink.cpp + test_encoder.cpp ) target_include_directories(itch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/tests/test_encoder.cpp b/tests/test_encoder.cpp new file mode 100644 index 0000000..b9dd532 --- /dev/null +++ b/tests/test_encoder.cpp @@ -0,0 +1,125 @@ +#include + +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/parser.hpp" +#include "itch/replay.hpp" +#include "itch/venue.hpp" + +namespace { + +auto fill_stock(char (&dest)[itch::STOCK_LEN], std::string_view symbol) -> void { + std::memset(dest, ' ', itch::STOCK_LEN); + std::memcpy( + dest, symbol.data(), std::min(symbol.size(), static_cast(itch::STOCK_LEN)) + ); +} + +// Round-trips a message through encode -> parse and returns the decoded variant. +auto round_trip(const itch::Message& message) -> itch::Message { + const std::vector frame = itch::encode_frame(message); + itch::Parser parser; + std::vector out = parser.parse(std::span {frame}); + EXPECT_EQ(out.size(), 1U); + return out.front(); +} + +} // namespace + +TEST(Encoder, AddOrderRoundTrips) { + itch::AddOrderMessage original {}; + original.stock_locate = 7; + original.tracking_number = 3; + original.timestamp = 0x0000123456789ABCULL & 0x0000FFFFFFFFFFFFULL; // fits 48 bits + original.order_reference_number = 987654321; + original.buy_sell_indicator = 'B'; + original.shares = 500; + fill_stock(original.stock, "AAPL"); + original.price = 1500000; + + const auto decoded = std::get(round_trip(itch::Message {original})); + EXPECT_EQ(decoded.stock_locate, original.stock_locate); + EXPECT_EQ(decoded.tracking_number, original.tracking_number); + EXPECT_EQ(decoded.timestamp, original.timestamp); + EXPECT_EQ(decoded.order_reference_number, original.order_reference_number); + EXPECT_EQ(decoded.buy_sell_indicator, 'B'); + EXPECT_EQ(decoded.shares, 500U); + EXPECT_EQ(itch::to_string(decoded.stock, itch::STOCK_LEN), "AAPL"); + EXPECT_EQ(decoded.price, 1500000U); +} + +TEST(Encoder, NoiiRoundTripsAllFields) { + itch::NOIIMessage original {}; + original.stock_locate = 11; + original.timestamp = 42; + original.paired_shares = 100000; + original.imbalance_shares = 2500; + original.imbalance_direction = 'B'; + fill_stock(original.stock, "MSFT"); + original.far_price = 3000000; + original.near_price = 3000100; + original.current_reference_price = 3000050; + original.cross_type = 'O'; + original.price_variation_indicator = 'L'; + + const auto decoded = std::get(round_trip(itch::Message {original})); + EXPECT_EQ(decoded.paired_shares, 100000U); + EXPECT_EQ(decoded.imbalance_shares, 2500U); + EXPECT_EQ(decoded.imbalance_direction, 'B'); + EXPECT_EQ(itch::to_string(decoded.stock, itch::STOCK_LEN), "MSFT"); + EXPECT_EQ(decoded.near_price, 3000100U); + EXPECT_EQ(decoded.cross_type, 'O'); +} + +TEST(Encoder, FrameCarriesLengthPrefix) { + itch::SystemEventMessage event {}; + event.event_code = 'O'; + const auto frame = itch::encode_frame(itch::Message {event}); + const auto body = itch::encode_message(itch::Message {event}); + // The 2-byte length prefix should equal the body size. + const auto length = + (static_cast(frame[0]) << 8) | static_cast(frame[1]); + EXPECT_EQ(length, body.size()); + EXPECT_EQ(frame.size(), body.size() + 2); +} + +TEST(Replay, DeliversAllMessagesAndScalesTiming) { + // Two messages 100 ms apart in feed time; at 100x speed the wall gap is ~1 ms. + itch::AddOrderMessage first {}; + first.stock_locate = 1; + first.timestamp = 0; + fill_stock(first.stock, "AAA"); + itch::AddOrderMessage second = first; + second.timestamp = 100'000'000; // 100 ms later + + std::vector stream; + for (const auto& msg : {first, second}) { + const auto frame = itch::encode_frame(itch::Message {msg}); + stream.insert(stream.end(), frame.begin(), frame.end()); + } + + itch::ReplayEngine engine {100.0}; + int count = 0; + const auto start = std::chrono::steady_clock::now(); + const auto replayed = + engine.replay(std::span {stream}, [&](const itch::Message&) { ++count; }); + const auto elapsed = std::chrono::steady_clock::now() - start; + + EXPECT_EQ(replayed, 2U); + EXPECT_EQ(count, 2); + // 100 ms / 100x = ~1 ms expected; assert it paced at least a little but not + // anywhere near the full 100 ms (generous bounds to avoid flakiness). + EXPECT_LT(elapsed, std::chrono::milliseconds {80}); +} + +TEST(Venue, Nasdaq50PolicyEnumeratesMessageTypes) { + int count = 0; + itch::venue::Nasdaq50::for_each_message_type([&count](char) { ++count; }); + EXPECT_EQ(count, 23); // The full ITCH 5.0 message set. + EXPECT_EQ(itch::venue::Nasdaq50::name(), "NASDAQ TotalView-ITCH 5.0"); +} From b340fe5bc1097f824387851724d21f785eb7fe88 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Mon, 29 Jun 2026 01:40:07 +0100 Subject: [PATCH 031/144] Make examples and itch-tool build under C++20 std::println is C++23-only; the examples and the CLI used it, breaking the C++20 CI leg. Switch them to std::format (which is C++20) written to std::cout/std::cerr, via a small print_line helper in the CLI. The library itself was already C++20 clean. Verified both the C++20 and C++23 builds and test suites pass. Co-Authored-By: Claude Opus 4.8 --- examples/analytics/vwap_example.cpp | 11 ++++--- examples/book/book_engine_example.cpp | 13 ++++---- examples/transport/pcap_replay_example.cpp | 17 +++++----- tools/itch_tool/main.cpp | 37 +++++++++++++--------- 4 files changed, 45 insertions(+), 33 deletions(-) diff --git a/examples/analytics/vwap_example.cpp b/examples/analytics/vwap_example.cpp index cf596a5..3ff3877 100644 --- a/examples/analytics/vwap_example.cpp +++ b/examples/analytics/vwap_example.cpp @@ -1,5 +1,6 @@ +#include #include -#include +#include #include #include "itch/analytics/vwap.hpp" @@ -10,13 +11,13 @@ // trade tape through a running Vwap accumulator. auto main(int argc, char* argv[]) -> int { if (argc < 3) { - std::println(stderr, "Usage: {} ", argv[0]); + std::cerr << std::format("Usage: {} \n", argv[0]); return 1; } std::ifstream file {argv[1], std::ios::binary}; if (!file) { - std::println(stderr, "Error: cannot open '{}'.", argv[1]); + std::cerr << std::format("Error: cannot open '{}'.\n", argv[1]); return 1; } std::vector buffer {std::istreambuf_iterator {file}, {}}; @@ -36,6 +37,8 @@ auto main(int argc, char* argv[]) -> int { manager.process(msg); }); - std::println("VWAP for {}: {:.4f} over {} shares", argv[2], vwap.value(), vwap.volume()); + std::cout << std::format( + "VWAP for {}: {:.4f} over {} shares\n", argv[2], vwap.value(), vwap.volume() + ); return 0; } diff --git a/examples/book/book_engine_example.cpp b/examples/book/book_engine_example.cpp index 2aa4348..d9c5c19 100644 --- a/examples/book/book_engine_example.cpp +++ b/examples/book/book_engine_example.cpp @@ -1,22 +1,23 @@ #include +#include #include -#include +#include #include #include "itch/book/book_manager.hpp" #include "itch/parser.hpp" // Reconstructs every symbol's order book in one pass over an ITCH file using the -// multi-symbol BookManager, printing best-bid/offer changes and the trade tape. +// multi-symbol BookManager, counting best-bid/offer changes and trades. auto main(int argc, char* argv[]) -> int { if (argc < 2) { - std::println(stderr, "Usage: {} [symbol]", argv[0]); + std::cerr << std::format("Usage: {} [symbol]\n", argv[0]); return 1; } std::ifstream file {argv[1], std::ios::binary}; if (!file) { - std::println(stderr, "Error: cannot open '{}'.", argv[1]); + std::cerr << std::format("Error: cannot open '{}'.\n", argv[1]); return 1; } std::vector buffer {std::istreambuf_iterator {file}, {}}; @@ -38,8 +39,8 @@ auto main(int argc, char* argv[]) -> int { manager.process(msg); }); - std::println( - "Reconstructed {} books, {} BBO changes, {} trades.", + std::cout << std::format( + "Reconstructed {} books, {} BBO changes, {} trades.\n", manager.book_count(), bbo_updates, trades diff --git a/examples/transport/pcap_replay_example.cpp b/examples/transport/pcap_replay_example.cpp index 2bf6574..5407d88 100644 --- a/examples/transport/pcap_replay_example.cpp +++ b/examples/transport/pcap_replay_example.cpp @@ -1,5 +1,6 @@ #include -#include +#include +#include #include #include "itch/messages.hpp" @@ -10,7 +11,7 @@ // messages, with sequence gaps surfaced through the embedded tracker. auto main(int argc, char* argv[]) -> int { if (argc < 2) { - std::println(stderr, "Usage: {} [udp_port]", argv[0]); + std::cerr << std::format("Usage: {} [udp_port]\n", argv[0]); return 1; } @@ -19,14 +20,14 @@ auto main(int argc, char* argv[]) -> int { ++message_count; const char type = std::visit([](const auto& msg) { return msg.message_type; }, message); if (message_count <= 10) { - std::println("message {:>3}: type '{}'", message_count, type); + std::cout << std::format("message {:>3}: type '{}'\n", message_count, type); } }}; reader.mold_decoder().tracker().set_gap_callback( [](std::string_view session, std::uint64_t expected, std::uint64_t received) { - std::println( - stderr, "gap on session '{}': expected {} but got {}", session, expected, received + std::cerr << std::format( + "gap on session '{}': expected {} but got {}\n", session, expected, received ); } ); @@ -36,12 +37,12 @@ auto main(int argc, char* argv[]) -> int { } if (!reader.read_file(argv[1])) { - std::println(stderr, "Error: '{}' is not a readable pcap/pcapng capture.", argv[1]); + std::cerr << std::format("Error: '{}' is not a readable pcap/pcapng capture.\n", argv[1]); return 1; } - std::println( - "Decoded {} ITCH messages from {} UDP datagrams ({} missing messages detected).", + std::cout << std::format( + "Decoded {} ITCH messages from {} UDP datagrams ({} missing messages detected).\n", reader.messages_decoded(), reader.udp_datagrams(), reader.mold_decoder().tracker().gap_count() diff --git a/tools/itch_tool/main.cpp b/tools/itch_tool/main.cpp index 889422a..ca4a6b3 100644 --- a/tools/itch_tool/main.cpp +++ b/tools/itch_tool/main.cpp @@ -1,9 +1,9 @@ #include #include #include +#include #include #include -#include #include #include #include @@ -15,6 +15,13 @@ namespace { +// Writes a formatted line to a stream. Uses std::format (C++20) rather than +// std::println (C++23) so the tool builds on the project's full standard range. +template +auto print_line(std::ostream& out, std::string_view fmt, const Args&... args) -> void { + out << std::vformat(fmt, std::make_format_args(args...)) << '\n'; +} + // Reads an entire file into a byte buffer. auto read_file(const std::string& path) -> std::vector { std::ifstream file {path, std::ios::binary}; @@ -63,13 +70,13 @@ auto cmd_stats(std::span data) -> int { ++counts[static_cast(message_type_of(msg))]; ++total; }); - std::println("{:<6} {:>14}", "type", "count"); + print_line(std::cout, "{:<6} {:>14}", "type", "count"); for (std::size_t type = 0; type < counts.size(); ++type) { if (counts[type] > 0) { - std::println("{:<6} {:>14}", static_cast(type), counts[type]); + print_line(std::cout, "{:<6} {:>14}", static_cast(type), counts[type]); } } - std::println("{:<6} {:>14}", "TOTAL", total); + print_line(std::cout, "{:<6} {:>14}", "TOTAL", total); return 0; } @@ -81,7 +88,7 @@ auto cmd_inspect(std::span data, std::uint64_t limit) -> int { ++shown; } }); - std::println("(showed {} messages)", shown); + print_line(std::cout, "(showed {} messages)", shown); return 0; } @@ -95,7 +102,7 @@ auto cmd_filter_or_convert( if (!out_path.empty()) { file_out.open(out_path, std::ios::binary); if (!file_out) { - std::println(stderr, "Error: cannot open output '{}'.", out_path); + print_line(std::cerr, "Error: cannot open output '{}'.", out_path); return 1; } } @@ -107,18 +114,18 @@ auto cmd_filter_or_convert( } }); sink.flush(); - std::println(stderr, "Wrote {} rows.", sink.rows_written()); + print_line(std::cerr, "Wrote {} rows.", sink.rows_written()); return 0; } auto usage(const char* program) -> int { - std::println(stderr, "itch-tool - inspect, filter, and convert NASDAQ ITCH 5.0 data"); - std::println(stderr, "Usage:"); - std::println(stderr, " {} stats ", program); - std::println(stderr, " {} inspect [--limit N]", program); - std::println(stderr, " {} filter --types [--out ]", program); - std::println(stderr, " {} convert [--to csv] [--out ]", program); - std::println(stderr, "Input may be a raw ITCH stream or a .pcap/.pcapng capture."); + print_line(std::cerr, "itch-tool - inspect, filter, and convert NASDAQ ITCH 5.0 data"); + print_line(std::cerr, "Usage:"); + print_line(std::cerr, " {} stats ", program); + print_line(std::cerr, " {} inspect [--limit N]", program); + print_line(std::cerr, " {} filter --types [--out ]", program); + print_line(std::cerr, " {} convert [--to csv] [--out ]", program); + print_line(std::cerr, "Input may be a raw ITCH stream or a .pcap/.pcapng capture."); return 1; } @@ -153,7 +160,7 @@ auto main(int argc, char* argv[]) -> int { const std::vector data = read_file(path); if (data.empty()) { - std::println(stderr, "Error: cannot read '{}' (missing or empty).", path); + print_line(std::cerr, "Error: cannot read '{}' (missing or empty).", path); return 1; } const std::span view {data}; From 7f3d2887bfb4867ee86db31355dd3da823d5a460 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Mon, 29 Jun 2026 02:01:43 +0100 Subject: [PATCH 032/144] Add Doxygen docs site published to GitHub Pages - docs/Doxyfile: Doxygen config using the README as the landing page (USE_MDFILE_AS_MAINPAGE), GitHub-style markdown anchors, tree-view sidebar, and the modern doxygen-awesome-css theme; the other top-level markdown docs are included as pages so cross-links resolve. - docs/awesome-head.html: the theme enhancements (dark-mode toggle, fragment copy buttons, paragraph links, interactive TOC) injected into the generated header. - .github/workflows/docs.yml: builds the docs with the runner's Doxygen (header generated to match its version), clones the pinned theme, and deploys to GitHub Pages on every push to main (plus workflow_dispatch). - build_documentation(): the local `docs` target now runs the committed Doxyfile and auto-fetches the theme (cmake/FetchDocsTheme.cmake) so local and published docs match. Replaces the broken Doxyfile.in/doxygen_add_docs path. - README docs badge/link, CHANGELOG and CONTRIBUTING entries; generated docs and the vendored theme are gitignored. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/docs.yml | 80 +++++++++++++++++++++++++++++++ .gitignore | 6 +++ CHANGELOG.md | 7 +++ CONTRIBUTING.md | 4 ++ README.md | 3 ++ cmake/FetchDocsTheme.cmake | 22 +++++++++ cmake/Helpers.cmake | 28 +++++++---- docs/Doxyfile | 96 ++++++++++++++++++++++++++++++++++++++ docs/awesome-head.html | 13 ++++++ 9 files changed, 249 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 cmake/FetchDocsTheme.cmake create mode 100644 docs/Doxyfile create mode 100644 docs/awesome-head.html diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..1e3101b --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,80 @@ +name: docs + +# Build the Doxygen documentation and publish it to GitHub Pages on every push to +# main. The site uses the README as its landing page and the modern +# doxygen-awesome-css theme with a dark-mode toggle, copy buttons, and an +# interactive table of contents. + +on: + push: + branches: ["main"] + workflow_dispatch: + +# Allow the workflow to publish to GitHub Pages. +permissions: + contents: read + pages: write + id-token: write + +# Only one Pages deployment at a time; let an in-progress run finish. +concurrency: + group: "pages" + cancel-in-progress: false + +env: + DOXYGEN_AWESOME_VERSION: "v2.3.4" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Install Doxygen and Graphviz + run: sudo apt-get update && sudo apt-get install -y doxygen graphviz + + - name: Fetch the doxygen-awesome-css theme + run: | + git clone --depth 1 --branch "$DOXYGEN_AWESOME_VERSION" \ + https://github.com/jothepro/doxygen-awesome-css.git docs/doxygen-awesome-css + + - name: Generate a theme-aware HTML header + run: | + # Generate the default header for THIS Doxygen version, then inject the + # doxygen-awesome enhancements (docs/awesome-head.html) before . + # Generating the header with the runner's own Doxygen avoids template + # token mismatches across versions. + doxygen -w html docs/header.html docs/footer.html docs/_awesome_unused.css docs/Doxyfile + awk '/<\/head>/{while((getline line < "docs/awesome-head.html") > 0) print line} {print}' \ + docs/header.html > docs/header.tmp + mv docs/header.tmp docs/header.html + + - name: Build documentation + run: | + # Append the header and Graphviz settings to the committed Doxyfile, then + # build from stdin so the relative INPUT paths resolve from the repo root. + { + cat docs/Doxyfile + echo "HTML_HEADER = docs/header.html" + echo "HAVE_DOT = YES" + echo "DOT_IMAGE_FORMAT = svg" + echo "DOT_TRANSPARENT = YES" + } | doxygen - + touch docs/html/.nojekyll + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/html + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 5bf5345..33fe865 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,9 @@ scripts.sh # Generated coverage reports must not be committed; they belong in the build tree. coverage.html docs/coverage.html + +# Generated documentation and the vendored theme (cloned at build/publish time). +docs/html/ +docs/doxygen-awesome-css/ +docs/header.html +docs/footer.html diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dbafeb..440e06a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Documentation site**: a Doxygen configuration (`docs/Doxyfile`) using the + README as the landing page and the modern doxygen-awesome-css theme (dark-mode + toggle, copy buttons, interactive table of contents, tree-view sidebar), plus a + GitHub Actions workflow (`.github/workflows/docs.yml`) that builds and publishes + it to GitHub Pages on every push to `main`. The local `docs` CMake target + (`-DITCH_BUILD_DOCUMENTATION=ON`) builds the same styled documentation. + - **Phase 5 — Simulation & ecosystem** ([FEATURES.md](FEATURES.md)): - Replay engine (`itch/replay.hpp`): `ReplayEngine` drives a callback paced by the messages' nanosecond timestamps at original or scaled wall-clock speed, for diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e04d074..326cbc7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,6 +27,10 @@ Optional features are off by default and enabled with: vcpkg feature: `-DVCPKG_MANIFEST_FEATURES=python`). - `-DITCH_WITH_ARROW=ON` — Apache Arrow / Parquet export (needs the `arrow` vcpkg feature: `-DVCPKG_MANIFEST_FEATURES=arrow`). +- `-DITCH_BUILD_DOCUMENTATION=ON` — the Doxygen `docs` target. Build it with + `cmake --build build --target docs` (needs Doxygen; the modern theme is fetched + automatically). Output is written to `docs/html`. The same documentation is + published to GitHub Pages from `main` by `.github/workflows/docs.yml`. ## Code style diff --git a/README.md b/README.md index d19b3e0..280cc2c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # High-Performance NASDAQ ITCH 5.0 Parser [![Build](https://github.com/bbalouki/itchcpp/actions/workflows/build.yml/badge.svg)](https://github.com/bbalouki/itchcpp/actions/workflows/build.yml) +[![Docs](https://github.com/bbalouki/itchcpp/actions/workflows/docs.yml/badge.svg)](https://bbalouki.github.io/itchcpp/) ![Linux](https://img.shields.io/badge/Linux-FCC624?logo=linux&logoColor=black) ![macOS](https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=white) ![Windows](https://img.shields.io/badge/Windows-0078D6?logo=windows&logoColor=white) @@ -14,6 +15,8 @@ A modern, high-performance C++20 library for parsing NASDAQ TotalView-ITCH 5.0 protocol data feeds. This parser is designed for maximum speed, minimal memory overhead, and type safety, making it ideal for latency-sensitive financial applications, market data analysis, and quantitative research. +📖 **API documentation:** (generated with Doxygen, published on every push to `main`). + --- ## Table of Contents diff --git a/cmake/FetchDocsTheme.cmake b/cmake/FetchDocsTheme.cmake new file mode 100644 index 0000000..85c15ac --- /dev/null +++ b/cmake/FetchDocsTheme.cmake @@ -0,0 +1,22 @@ +# Clones the doxygen-awesome-css theme into the docs tree if it is absent, so the +# local `docs` target produces the same modern styling as the published site. The +# checkout is gitignored. Invoked by the `docs` custom target with -P. + +if(EXISTS "${THEME_DIR}/doxygen-awesome.css") + return() +endif() + +if(NOT GIT_EXECUTABLE) + message(WARNING "Git not found; docs will use the default Doxygen theme.") + return() +endif() + +message(STATUS "Fetching doxygen-awesome-css (${THEME_TAG}) into ${THEME_DIR} ...") +execute_process( + COMMAND "${GIT_EXECUTABLE}" clone --depth 1 --branch "${THEME_TAG}" + https://github.com/jothepro/doxygen-awesome-css.git "${THEME_DIR}" + RESULT_VARIABLE clone_result +) +if(NOT clone_result EQUAL 0) + message(WARNING "Could not fetch doxygen-awesome-css; docs will use the default theme.") +endif() diff --git a/cmake/Helpers.cmake b/cmake/Helpers.cmake index 77e2b63..3a6fa97 100644 --- a/cmake/Helpers.cmake +++ b/cmake/Helpers.cmake @@ -141,19 +141,27 @@ function(build_documentation) find_package(Doxygen) if (Doxygen_FOUND) message( - STATUS - "Found Doxygen Version " "${DOXYGEN_VERSION} " + STATUS + "Found Doxygen Version " "${DOXYGEN_VERSION} " "at ${DOXYGEN_EXECUTABLE}" ) - configure_file( - "${PROJECT_SOURCE_DIR}/docs/Doxyfile.in" - "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" - @ONLY - ) - doxygen_add_docs( - docs - CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" + find_package(Git QUIET) + # Run Doxygen on the committed docs/Doxyfile from the source root so its + # relative INPUT paths (README.md, include, src) resolve. The same + # Doxyfile is used by the GitHub Pages workflow, keeping local and + # published documentation identical. The modern theme is fetched first so + # local docs are styled like the published site. + add_custom_target( + docs + COMMAND ${CMAKE_COMMAND} + -DTHEME_DIR=${PROJECT_SOURCE_DIR}/docs/doxygen-awesome-css + -DTHEME_TAG=v2.3.4 + -DGIT_EXECUTABLE=${GIT_EXECUTABLE} + -P ${PROJECT_SOURCE_DIR}/cmake/FetchDocsTheme.cmake + COMMAND ${DOXYGEN_EXECUTABLE} "${PROJECT_SOURCE_DIR}/docs/Doxyfile" + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" COMMENT "Generating docs with Doxygen ..." + VERBATIM ) message(STATUS "Add 'docs' target") endif() diff --git a/docs/Doxyfile b/docs/Doxyfile new file mode 100644 index 0000000..262df8e --- /dev/null +++ b/docs/Doxyfile @@ -0,0 +1,96 @@ +# Doxygen configuration for ITCHCPP. +# +# Run from the repository root: `doxygen docs/Doxyfile` (the CMake `docs` target +# does this for you). Output is written to docs/html. The modern look comes from +# doxygen-awesome-css, referenced below; the GitHub Pages workflow clones it into +# docs/doxygen-awesome-css and generates a matching HTML header for the active +# Doxygen version. + +#--------------------------------------------------------------------------- +# Project +#--------------------------------------------------------------------------- +PROJECT_NAME = "ITCHCPP" +PROJECT_NUMBER = 1.6.0 +PROJECT_BRIEF = "High-Performance NASDAQ TotalView-ITCH 5.0 Parser & Market-Data Platform" +OUTPUT_DIRECTORY = docs +HTML_OUTPUT = html +CREATE_SUBDIRS = NO +FULL_PATH_NAMES = NO +JAVADOC_AUTOBRIEF = YES +QT_AUTOBRIEF = YES +TAB_SIZE = 4 +MARKDOWN_SUPPORT = YES +# GitHub-style heading anchors so the README's table-of-contents links resolve. +MARKDOWN_ID_STYLE = GITHUB +TOC_INCLUDE_HEADINGS = 4 +AUTOLINK_SUPPORT = YES +BUILTIN_STL_SUPPORT = YES + +#--------------------------------------------------------------------------- +# Build / extraction +#--------------------------------------------------------------------------- +EXTRACT_ALL = YES +EXTRACT_STATIC = YES +EXTRACT_LOCAL_CLASSES = YES +RECURSIVE = YES +SORT_MEMBER_DOCS = NO +QUIET = YES +WARN_IF_UNDOCUMENTED = NO + +#--------------------------------------------------------------------------- +# Input +#--------------------------------------------------------------------------- +INPUT = README.md \ + FEATURES.md \ + IMPROVEMENTS.md \ + CONTRIBUTING.md \ + CHANGELOG.md \ + python/README.md \ + include \ + src +FILE_PATTERNS = *.hpp *.h *.cpp *.md +USE_MDFILE_AS_MAINPAGE = README.md +EXCLUDE_PATTERNS = */build/* */build-*/* */.venv/* + +#--------------------------------------------------------------------------- +# Source browsing +#--------------------------------------------------------------------------- +SOURCE_BROWSER = YES +REFERENCED_BY_RELATION = YES +REFERENCES_RELATION = YES + +#--------------------------------------------------------------------------- +# Output formats +#--------------------------------------------------------------------------- +GENERATE_HTML = YES +GENERATE_LATEX = NO +GENERATE_XML = NO + +#--------------------------------------------------------------------------- +# HTML appearance - modern theme (doxygen-awesome-css) +#--------------------------------------------------------------------------- +# doxygen-awesome requires the light base color style and works best with a +# tree view sidebar. The extra stylesheets and JS files are provided by the +# cloned doxygen-awesome-css checkout; the dark-mode toggle, copy buttons, and +# interactive table of contents are wired up by the generated HTML header. +GENERATE_TREEVIEW = YES +DISABLE_INDEX = NO +FULL_SIDEBAR = NO +HTML_COLORSTYLE = LIGHT +HTML_DYNAMIC_SECTIONS = YES +HTML_COPY_CLIPBOARD = NO +HTML_EXTRA_STYLESHEET = docs/doxygen-awesome-css/doxygen-awesome.css \ + docs/doxygen-awesome-css/doxygen-awesome-sidebar-only.css \ + docs/doxygen-awesome-css/doxygen-awesome-sidebar-only-darkmode-toggle.css +HTML_EXTRA_FILES = docs/doxygen-awesome-css/doxygen-awesome-darkmode-toggle.js \ + docs/doxygen-awesome-css/doxygen-awesome-fragment-copy-button.js \ + docs/doxygen-awesome-css/doxygen-awesome-paragraph-link.js \ + docs/doxygen-awesome-css/doxygen-awesome-interactive-toc.js + +#--------------------------------------------------------------------------- +# Diagrams (graphviz; optional, enabled when dot is available) +#--------------------------------------------------------------------------- +HAVE_DOT = NO +CLASS_GRAPH = YES +COLLABORATION_GRAPH = NO +UML_LOOK = YES diff --git a/docs/awesome-head.html b/docs/awesome-head.html new file mode 100644 index 0000000..8e55835 --- /dev/null +++ b/docs/awesome-head.html @@ -0,0 +1,13 @@ + + + + + + From 56817bd5f9a748fc82f25e5ba864c674f52f8a74 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Sat, 4 Jul 2026 23:13:15 +0100 Subject: [PATCH 033/144] remove CLAUDE.md --- CLAUDE.md | 89 ------------------------------------------------------- 1 file changed, 89 deletions(-) delete mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index ca8a03f..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,89 +0,0 @@ -# Modern C++ Best Practices - -## Language & Compiler - -- Target C++17 minimum; prefer C++20/23. Set `CMAKE_CXX_STANDARD` explicitly; disable extensions (`-pedantic`). -- Enable `-Wall -Wextra -Wconversion -Wsign-conversion` and treat warnings as errors (`-Werror`). - -## Style & Naming - -- `m_` prefix for private members; `snake_case` for variables/functions; `PascalCase` for types; `SCREAMING_SNAKE_CASE` for macros and Constants. -- All symbols (variable, parameters, functions) must be at least 3 characters long. -- Alwayse Use `std::print`/`std::println`/`std::format` for output. -- No magic literals, replace with named `constexpr` values. Declare variables close to first use. -- Keep functions short and flat; remove unused variables, parameters, and includes. -- Use `#pragma once`; keep headers to declarations and inline/template definitions only. -- Do not use mdash (`--`) in code, comments, or documentation. - -## Variables & Types - -- Always initialize variables at declaration; prefer brace initialization `T x{value}` to prevent narrowing. -- Use `auto` when type is obvious; `const` by default; `constexpr` for compile-time constants. -- Avoid `using namespace std` in headers. Use `enum class`; use `std::string_view` for read-only strings. -- Never mix signed/unsigned arithmetic; never use C-style casts, use `static_cast`, `dynamic_cast`, etc. -- Never use raw pointers for ownership; use references or smart pointers instead. Avoid `void*` and `reinterpret_cast`. -- Never use `goto` or `longjmp`; prefer structured control flow and RAII for cleanup. -- Never use C-style arrays or C-style strings; prefer `std::array`, `std::vector`, and `std::string` for safety and convenience. - -## Memory Management - -- Follow RAII: tie every resource (memory, file, mutex) to an object's lifetime. -- Use `std::unique_ptr`/`std::shared_ptr`/`std::weak_ptr` via `make_unique`/`make_shared`, never `new`/`delete` directly. -- Follow Rule of Zero (all RAII members) or Rule of Five (when owning a raw resource). -- Detect leaks with AddressSanitizer; use standard containers over raw arrays. - -## Functions & Classes - -- One logical operation per function; always use trailing return types. -- Pass non-trivial types as `const T&`; return by value (rely on NRVO/RVO). -- Mark `[[nodiscard]]` where return values must not be ignored; mark `noexcept` when applicable. -- Single-argument constructors must be `explicit`; keep members `private`; use `override`/`final` on overrides. -- Prefer composition over inheritance; declare base destructors `virtual` (or `protected` non-virtual). -- Do not call virtual functions from constructors or destructors. - -## Error Handling - -- Use exceptions for errors (not for normal control flow); catch by `const` reference to avoid slicing. -- Maintain a consistent exception-safety guarantee: basic, strong, or no-throw. -- Use `static_assert` for compile-time invariants; `assert()` for debug-only preconditions, never on user input. -- Consider `std::expected` (C++23) for predictable, performance-sensitive failure paths. - -## Templates & Concurrency - -- Constrain templates with C++20 concepts; use `if constexpr` over SFINAE/`enable_if`. -- Use variadic templates and fold expressions instead of recursive specializations. -- Protect shared mutable state with `std::mutex`; prefer `std::jthread` over `std::thread`. -- Use `std::atomic` for lock-free single-variable operations; never `volatile` for inter-thread communication. -- Minimize lock duration; avoid callbacks or virtual calls while holding a lock; prefer `std::async` and parallel algorithms. - -## Performance & Build - -- Profile before optimizing; prefer `std::vector` for cache-friendly access; use `reserve()` when size is known. -- Use move semantics to avoid deep copies; prefer standard algorithms (`std::sort`, `std::transform`) over hand-written loops. -- Use CMake (≥ 3.25) with target-centric paradigm: `target_include_directories`, `target_compile_options`, `target_link_libraries`. -- Use `FetchContent` or vcpkg/Conan for dependencies; use out-of-source builds and CMake Presets. -- Enable `CMAKE_EXPORT_COMPILE_COMMANDS=ON` for tooling; use `ccache`/`sccache` to speed up rebuilds. - -## Testing & Tooling - -- Use GoogleTest/Catch2 with CTest; write tests before or alongside implementation. -- Run tests under AddressSanitizer and UndefinedBehaviourSanitizer; use fuzz testing for parsers and untrusted input. -- Enforce clang-format in CI; use clang-tidy with `cppcoreguidelines-*`, `modernize-*`, `readability-*` checks. -- Use debuggers (GDB, LLDB) actively rather than `std::cout`; use `compile_commands.json` for accurate tooling. -- Use real implementations; only mock external dependencies (LLM APIs, cloud services). -- Test public interface behavior, not implementation details, keeps tests resilient to refactoring. -- Tests must be fast, isolated, and have descriptive names; cover edge cases and error conditions. -- Location: `tests/unittests/` following source structure. - -## CI/CD & Maintenance - -## Versioning & Breaking Changes - -- Follow Semantic Versioning 2.0.0 (`MAJOR.MINOR.PATCH`). -- A breaking change is any backward-incompatible modification to the public API, including data schemas, CLI, and server communication formats. -- CI must build and test before merging; build across GCC, Clang, and MSVC. -- Document the _why_, not the _what_; use Doxygen (`///`) for API docs and `//` for implementation notes. -- Keep public API surface minimal; mark deprecated APIs with `[[deprecated("reason")]]` and include a migration path. -- Follow the Boy Scout Rule; track technical debt explicitly; maintain a `CHANGELOG.md` and `CONTRIBUTING.md`. - ---- From 19b2ffedc8b7b354a262d8e80a6f22f9fd219592 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:38:50 +0100 Subject: [PATCH 034/144] gitignore: track packaging, ignore python build artifacts --- .gitignore | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 33fe865..1c7eee4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,14 +13,27 @@ scratch_* scripts.sh *Presets.json .cache/ -/packaging +prompts.txt # Generated coverage reports must not be committed; they belong in the build tree. coverage.html docs/coverage.html -# Generated documentation and the vendored theme (cloned at build/publish time). +# Python build artifacts. The compiled extension (itchcpp/_itchcpp*.pyd|.so) is a +# build output; the hand-written .pyi stub beside it is source and is kept. +__pycache__/ +*.pyd +*.so +*.egg-info/ +.pytest_cache/ +.mypy_cache/ +dist/ +wheelhouse/ + +# Generated documentation. The doxygen-awesome-css theme and the HTML header are +# fetched/generated into the CMake build tree by build_documentation(), so only the +# rendered site under docs/html is produced inside docs/. docs/html/ -docs/doxygen-awesome-css/ -docs/header.html -docs/footer.html +CLAUDE.md +PROMPTS.md +DEVELOPER_GUIDE.md From fa0264cb78c05492a7b2baca65f7c2886d8888e0 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:38:54 +0100 Subject: [PATCH 035/144] ci: point build workflow at new itchcpp package layout --- .github/workflows/build.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f920265..7baca6f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -160,10 +160,6 @@ jobs: uses: lukka/run-vcpkg@v11 - name: Configure CMake (ASAN + UBSAN) - # The ITCH message structs are deliberately `#pragma pack(1)` to match the - # wire layout, so their multi-byte members are intentionally unaligned. - # Unaligned loads are safe on the supported architectures, so the UBSAN - # alignment check is excluded; every other UBSAN check remains fatal. run: | cmake -S . -B $BUILD_DIR \ -DITCH_BUILD_TESTS=ON \ @@ -309,8 +305,10 @@ jobs: - name: Run binding tests run: | python -m pip install --upgrade pip pytest - cp $BUILD_DIR/python/itchcpp*.so python/tests/ - cd python/tests && python -m pytest -q + # Drop the freshly built extension into the package so `import itchcpp` + # resolves to python/itchcpp with its compiled _itchcpp module. + cp $BUILD_DIR/python/_itchcpp*.so python/itchcpp/ + cd python && python -m pytest tests -q benchmark-smoke: runs-on: ubuntu-latest From 788167c988861988d240c09356a9fbb099b5f19c Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:38:56 +0100 Subject: [PATCH 036/144] ci: build docs via CMake docs target --- .github/workflows/docs.yml | 37 ++----------------------------------- 1 file changed, 2 insertions(+), 35 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1e3101b..e676fb8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,29 +1,19 @@ name: docs -# Build the Doxygen documentation and publish it to GitHub Pages on every push to -# main. The site uses the README as its landing page and the modern -# doxygen-awesome-css theme with a dark-mode toggle, copy buttons, and an -# interactive table of contents. - on: push: branches: ["main"] workflow_dispatch: -# Allow the workflow to publish to GitHub Pages. permissions: contents: read pages: write id-token: write -# Only one Pages deployment at a time; let an in-progress run finish. concurrency: group: "pages" cancel-in-progress: false -env: - DOXYGEN_AWESOME_VERSION: "v2.3.4" - jobs: build: runs-on: ubuntu-latest @@ -34,33 +24,10 @@ jobs: - name: Install Doxygen and Graphviz run: sudo apt-get update && sudo apt-get install -y doxygen graphviz - - name: Fetch the doxygen-awesome-css theme - run: | - git clone --depth 1 --branch "$DOXYGEN_AWESOME_VERSION" \ - https://github.com/jothepro/doxygen-awesome-css.git docs/doxygen-awesome-css - - - name: Generate a theme-aware HTML header - run: | - # Generate the default header for THIS Doxygen version, then inject the - # doxygen-awesome enhancements (docs/awesome-head.html) before . - # Generating the header with the runner's own Doxygen avoids template - # token mismatches across versions. - doxygen -w html docs/header.html docs/footer.html docs/_awesome_unused.css docs/Doxyfile - awk '/<\/head>/{while((getline line < "docs/awesome-head.html") > 0) print line} {print}' \ - docs/header.html > docs/header.tmp - mv docs/header.tmp docs/header.html - - name: Build documentation run: | - # Append the header and Graphviz settings to the committed Doxyfile, then - # build from stdin so the relative INPUT paths resolve from the repo root. - { - cat docs/Doxyfile - echo "HTML_HEADER = docs/header.html" - echo "HAVE_DOT = YES" - echo "DOT_IMAGE_FORMAT = svg" - echo "DOT_TRANSPARENT = YES" - } | doxygen - + cmake -S . -B build -DITCH_BUILD_DOCUMENTATION=ON + cmake --build build --target docs touch docs/html/.nojekyll - name: Upload Pages artifact From 20ad2f87ed22cce6af88277c2c5fbd3ba342a958 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:04 +0100 Subject: [PATCH 037/144] ci: add wheel build and PyPI publish workflow --- .github/workflows/publish.yml | 70 +++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..550dbb6 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,70 @@ +name: Publish + +on: + push: + tags: + - "v*" + workflow_dispatch: + +jobs: + build_wheels: + name: Wheels (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-13, macos-14] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.3 + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: ./wheelhouse/*.whl + + build_sdist: + name: Source distribution + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build sdist + run: pipx run build --sdist + + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + publish: + name: Publish to PyPI and create the release + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + permissions: + contents: write + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist + password: ${{ secrets.ITCHCPP_PYPI_API_TOKEN }} + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + files: dist/* + generate_release_notes: true From d34a53f8f04a85bdbfcb6f5b3932bd4861afb9ed Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:10 +0100 Subject: [PATCH 038/144] docs: remove CHANGELOG.md --- CHANGELOG.md | 168 --------------------------------------------------- 1 file changed, 168 deletions(-) delete mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 440e06a..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,168 +0,0 @@ -# Changelog - -All notable changes to this project are documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [Unreleased] - -### Added - -- **Documentation site**: a Doxygen configuration (`docs/Doxyfile`) using the - README as the landing page and the modern doxygen-awesome-css theme (dark-mode - toggle, copy buttons, interactive table of contents, tree-view sidebar), plus a - GitHub Actions workflow (`.github/workflows/docs.yml`) that builds and publishes - it to GitHub Pages on every push to `main`. The local `docs` CMake target - (`-DITCH_BUILD_DOCUMENTATION=ON`) builds the same styled documentation. - -- **Phase 5 — Simulation & ecosystem** ([FEATURES.md](FEATURES.md)): - - Replay engine (`itch/replay.hpp`): `ReplayEngine` drives a callback paced by - the messages' nanosecond timestamps at original or scaled wall-clock speed, for - realistic backtesting and system simulation. - - ITCH encoder / writer (`itch/encoder.hpp`): `encode_message` and `encode_frame` - serialize any `Message` back to valid wire bytes (the inverse of the parser), - guaranteeing `parse(encode(msg)) == msg`. This synthesizes streams for tests, - scenario generation, and golden fixtures, and backs the Python `to_bytes()`. - - Multi-venue / multi-version extension seam (`itch/venue.hpp`): a `VenuePolicy` - concept and the concrete `Nasdaq50` policy, so additional ITCH-like venues and - versions (BX/PSX, ITCH 4.1) can be added without rewriting the dispatch - machinery. Only the NASDAQ 5.0 policy is implemented. - - Packaging maturity: a Conan recipe (`conanfile.py`), a `CONTRIBUTING.md`, and a - documented versioning/ABI compatibility policy. - -- **Phase 4 — Research & interoperability** ([FEATURES.md](FEATURES.md)): - - CSV and streaming sinks (`itch/io/sink.hpp`, `itch/io/csv_sink.hpp`): a generic - `MessageSink` interface and a `CsvSink` that flattens every message type into a - normalized wide CSV table, with no dependencies. - - Apache Arrow / Parquet export (`itch/io/arrow_export.hpp`), gated behind the - optional `ITCH_WITH_ARROW` CMake option (Arrow via the `arrow` vcpkg feature); - turns a feed into a columnar table for pandas/Polars/DuckDB/Spark. Off by - default so the core stays dependency-free. - - `itch-tool` command-line utility (`tools/itch_tool`, `ITCH_BUILD_TOOLS`): the - `stats`, `inspect`, `filter`, and `convert` subcommands, auto-detecting raw - ITCH versus `.pcap`/`.pcapng` input. - - pybind11 Python bindings (`python/`, `ITCH_BUILD_PYTHON`): a native module that - mirrors the pure-Python `itch` package's `MessageParser` and message-class - API (typed messages, `decode_price`) so it can serve as a faster drop-in - backend, plus the book engine and VWAP. Includes a `pyproject.toml` - (scikit-build-core) and pytest tests. - - Per-message Doxygen documentation for every ITCH message struct and field in - `itch/messages.hpp`, sourced from the message semantics in the `itch` package. - -- **Phase 3 — Analytics & microstructure** ([FEATURES.md](FEATURES.md)). A header-only - `itch::analytics` layer driven off the Phase 2 trade tape and book: - - Bar builders (`analytics/bars.hpp`): OHLCV aggregation over time, tick, and - volume clocks via a single `BarBuilder` template parameterized by a clock policy. - - VWAP and TWAP (`analytics/vwap.hpp`): running and interval volume- and - time-weighted average prices. - - Microstructure metrics (`analytics/microstructure.hpp`): spread, mid price, - depth-at-level, top-of-book queue imbalance, and Cont-Kukanov-Stoikov order-flow - imbalance. - - NOII / imbalance (`analytics/imbalance.hpp`): `ImbalanceInfo` surfacing the - `I` message fields with strong price types and a direction description. - - Auction reconstruction (`analytics/auctions.hpp`): `AuctionTracker` pairs each - Cross Trade (`Q`) with the latest NOII (`I`) context to reconstruct opening, - closing, halt, and IPO crosses. - - Analytics tests and a VWAP example (`examples/analytics/vwap_example.cpp`). - -- **Phase 2 — Production order-book engine** ([FEATURES.md](FEATURES.md)). A - full-market book engine alongside the existing single-symbol `LimitOrderBook` - (which is unchanged and retained): - - `itch::book::L3Book` (`itch/book/l3_book.hpp`): an allocation-light, order-level - book. Orders live in a reusable object pool linked into intrusive FIFO queues, - price levels are flat sorted ladders, and order lookup by reference number uses - a flat open-addressed map (`itch/book/order_index.hpp`), removing the per-order - `std::shared_ptr`, list node, and map node of the original design. - - `itch::book::BookManager` (`itch/book/book_manager.hpp`): maintains a book per - symbol from one pass over the feed, routed by stock locate code in O(1), with an - optional symbol universe filter. - - Best-bid/offer change events (`set_bbo_callback`) and configurable L2 aggregated - and L3 order-level depth snapshots (`L3Book::depth`, `L3Book::orders_at`). - - Trade-tape extraction (`itch/tape.hpp`, `set_trade_callback`) from `E`/`C`/`P`/`Q` - messages, exposing the `printable` flag to distinguish displayable from hidden - prints. - - A zero-copy overlay parse API (`itch/overlay.hpp`): lazy typed views over the - raw buffer with on-access endianness conversion, for callers that touch only a - few fields per message. - - A book/overlay benchmark (`benchmarks/book_bench.cpp`) and a book-engine example - (`examples/book/book_engine_example.cpp`), with measured numbers in the README. - -- **Phase 1 — Real feed ingestion** ([FEATURES.md](FEATURES.md)). The library can - now consume ITCH as it is actually delivered, not only pre-stripped message - streams: - - `itch::transport::MoldUdp64Decoder` (`itch/transport/moldudp64.hpp`) decodes - the UDP multicast framing NASDAQ uses for live dissemination: it unpacks the - session, sequence number, and message blocks of each datagram and feeds them - to the existing parser, handling heartbeat and end-of-session packets. - - `itch::transport::SoupBinDecoder` (`itch/transport/soupbintcp.hpp`) decodes - the TCP framing used for the Glimpse snapshot and recovery/replay services, - including login, heartbeats, logout, end-of-session, and sequenced/unsequenced - data, buffering partial packets across stream segments. - - `itch::transport::PcapReader` (`itch/transport/pcap.hpp`) replays a feed - straight from a captured `.pcap`/`.pcapng` file. It is implemented in-house - with no libpcap dependency, walking the Ethernet/IPv4/IPv6/UDP layers and - feeding the payload through the MoldUDP64 decoder, with an optional UDP - destination-port filter. - - `itch::transport::SequenceTracker` and `RetransmitRequester` - (`itch/transport/sequencing.hpp`) track per-session sequence numbers across - the transport layer, surface gaps through a callback, and expose a re-request - hook a caller can implement to drive recovery against a replay server. - - A transport fuzz target (`fuzz/moldudp64_fuzzer.cpp`) and a `pcap` replay - example (`examples/transport/pcap_replay_example.cpp`). - -- `std::span` overloads of `Parser::parse` as the preferred, - size-carrying buffer interface. The `(const char*, size_t)` overloads remain as - thin shims for C interop. (IMPROVEMENTS 3.1) -- Non-throwing `Parser::try_parse` entry points returning - `std::expected<…, ParseError>` for latency-sensitive callers that prefer not to - use exceptions on the hot path. Available when the standard library provides - `std::expected` (C++23). (IMPROVEMENTS 3.2) -- An explicit error policy on `Parser`: `set_error_callback`, the - `unknown_message_count` / `malformed_message_count` diagnostics counters, and - `reset_diagnostics`. (IMPROVEMENTS 1.2) -- Per-message length validation: a frame whose declared length is smaller than its - message type requires is now rejected (counted, and reported to the error - callback) instead of being decoded into the following frame. (IMPROVEMENTS 1.1) -- Strongly typed fixed-point `Price` (`itch::StandardPrice`, `itch::MwcbPrice`) in - `itch/price.hpp`, encoding the decimal scale in the type so the 4-vs-8 decimal - distinction cannot be mixed up. (IMPROVEMENTS 3.3) -- Timestamp-to-wall-clock helpers in `itch/time.hpp` (`to_time_point`, - `format_timestamp`, `format_time_point`). (IMPROVEMENTS 3.4) -- Parser fuzz target (`fuzz/parser_fuzzer.cpp`, `ITCH_BUILD_FUZZERS`) and a - time-budgeted CI fuzzing job. (IMPROVEMENTS 5.4) -- Published, reproducible benchmark numbers in the README and a CI benchmark smoke - job. (IMPROVEMENTS 2.4) -- `scripts/fetch_sample.sh` to download an official ITCH sample on demand. - -### Changed - -- **Behavior change:** the library no longer writes to `std::cerr` on an unknown - message type. Unknown types are now silently skipped, counted, and surfaced - through the optional error callback. Callers that relied on the stderr output - should install an error callback or read the diagnostics counters. - (IMPROVEMENTS 1.2) -- Message dispatch now uses a compile-time, type-byte-indexed flat table instead of - a `std::map>`, removing the per-message tree lookup and - type-erased call. (IMPROVEMENTS 2.1) -- Byte swapping now uses `std::byteswap` / `std::bit_cast` instead of union - type-punning, eliminating undefined behavior on the hot path. (IMPROVEMENTS 1.3) -- Order-book rendering and indicator lookups now use `std::format` and `constexpr` - frozen tables instead of iostream manipulators and namespace-scope `std::map` - globals. (IMPROVEMENTS 4.2, 4.3) -- Standardized the minimum CMake version on 3.25 across the project. - (IMPROVEMENTS 5.2) -- README architecture section corrected to describe the parser as a fast, - allocation-free, single-pass field decoder rather than a "zero-copy / - zero-overhead" deserializer. (IMPROVEMENTS 2.2) - -### Fixed - -- Renamed misspelled public CMake helpers: `anable_address_sanitizer` → - `enable_address_sanitizer`, `build_documenation` → `build_documentation`, - `apply_formating` → `apply_formatting`, `apply_clang_tidy_globaly` → - `apply_clang_tidy_globally`. (IMPROVEMENTS 5.5) -- CI now runs on pull requests and adds clang-format, clang-tidy, ASAN/UBSAN, and - coverage jobs. (IMPROVEMENTS 5.3) -- Stopped tracking the generated `docs/coverage.html`; coverage is now generated - into the build tree only. (IMPROVEMENTS 5.1) From 6d68a1f075ab18f599950ef35d649363c8ec9d24 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:13 +0100 Subject: [PATCH 039/144] docs: remove FEATURES.md --- FEATURES.md | 153 ---------------------------------------------------- 1 file changed, 153 deletions(-) delete mode 100644 FEATURES.md diff --git a/FEATURES.md b/FEATURES.md deleted file mode 100644 index b1b3e22..0000000 --- a/FEATURES.md +++ /dev/null @@ -1,153 +0,0 @@ -# FEATURES - -This document is the product roadmap for ITCHCPP. Where [IMPROVEMENTS.md](IMPROVEMENTS.md) -is about making the existing parser correct, fast, and honest, this file is about -the new capabilities that turn a NASDAQ ITCH 5.0 file parser into a market-data -platform that quantitative teams, trading firms, and researchers can rely on -everywhere in their workflow. - -## Vision - -> A single, modern C++ library that ingests real exchange market data — live or -> historical — reconstructs the book, exposes microstructure analytics, and feeds -> both production trading systems and Python research notebooks, with the speed -> expected of latency-sensitive infrastructure. - -Today ITCHCPP parses the raw, length-prefixed ITCH 5.0 message stream and -reconstructs a single-symbol order book. To reach the vision it needs to read the -data formats that actually arrive on the wire and on disk, scale the book to whole -markets, compute the metrics quants ask for, and meet researchers in the tools -they already use. - -The roadmap is phased. Each phase is independently valuable and shippable; later -phases build on earlier ones. The order reflects what unlocks the most real-world -usage first. - ---- - -## Phase 1 — Real feed ingestion - -_Goal: make ITCHCPP usable on data as it actually exists, not just pre-stripped -message streams._ - -The current parser assumes a raw concatenation of length-prefixed messages. Real -ITCH is delivered inside transport framing. Without these decoders the library -cannot consume a live feed or most captured data. - -- **MoldUDP64 decoder** — the UDP multicast framing NASDAQ uses for live - dissemination. Unpacks sessions, sequence numbers, and message blocks, then - feeds the existing message parser. This is the gateway to live data. -- **SoupBinTCP decoder** — the TCP framing used for the glimpse snapshot and - recovery/replay services. Handles login, heartbeats, and sequenced payloads. -- **pcap ingestion** — replay market data straight from captured network traffic - (`.pcap`/`.pcapng`), the most common form in which firms archive and share feeds. -- **Gap detection and recovery hooks** — track sequence numbers across the - transport layer, surface gaps, and expose re-request hooks so a caller can drive - recovery against a replay server. - ---- - -## Phase 2 — Production order-book engine - -_Goal: reconstruct the full market, fast, not just one symbol._ - -The current `LimitOrderBook` is constructed for a single symbol -([include/itch/order_book.hpp:75](include/itch/order_book.hpp)) and is allocation- -heavy (see [IMPROVEMENTS.md](IMPROVEMENTS.md) 2.3). A serious book engine handles -every symbol on the feed at line rate. - -- **Multi-symbol book manager** — maintain books for all (or a selected universe - of) symbols from one pass over the feed, keyed by stock locate code for O(1) - routing. -- **Allocation-free internals** — order object pool / arena, intrusive FIFO queues - per price level, and flat price ladders, with O(1) lookup of orders by reference - number. Removes the per-order heap and refcount cost. (Deferred book rewrite - tracked from [IMPROVEMENTS.md](IMPROVEMENTS.md) 2.3; the current `std::shared_ptr` - book is correct but allocation-heavy.) -- **Zero-copy overlay parse API** — an alternative to the current single-pass field - decoder: typed views over the raw buffer with lazy, on-access endianness - conversion, for callers that touch only a few fields per message. The library - today decodes eagerly into host-order structs; this overlay is tracked from - [IMPROVEMENTS.md](IMPROVEMENTS.md) 2.2 as the stronger product direction. -- **BBO and top-of-book change events** — emit best-bid/offer updates as they - happen, the primary signal most consumers need. -- **Depth snapshots** — configurable L2 aggregated depth and full L3 order-level - detail, on demand or on every change. -- **Trade tape extraction** — a clean stream of executions, correctly - distinguishing displayable from non-displayable/hidden prints using the - printable flag. - ---- - -## Phase 3 — Analytics & microstructure - -_Goal: answer the questions quants actually ask, in the library, correctly._ - -Once the book and tape are reconstructed, the high-value layer is derived metrics. -Computing them centrally and correctly saves every downstream team from -reimplementing them. - -- **Bar builders** — OHLCV aggregation over time, volume, and tick clocks. -- **VWAP / TWAP** — running and interval volume- and time-weighted average prices. -- **Order-flow imbalance and microstructure metrics** — spread, depth at level, - queue imbalance, and order-flow imbalance, the staples of short-horizon research. -- **NOII / imbalance analytics** — surface the net order imbalance indicator data - the feed already carries in a usable form. -- **Auction reconstruction** — opening, closing, halt, and IPO cross price - discovery from the cross and NOII messages. - ---- - -## Phase 4 — Research & interoperability - -_Goal: meet quants where they live. This is what "everywhere in finance" means._ - -Most quantitative research happens in Python and columnar data tools. A -C++-only library, however fast, is invisible to that workflow until it has -bindings and standard export formats. - -- **Python bindings (pybind11)** — first-class, zero-friction access from Python, - exposing parsing, the book engine, and analytics. The single highest-leverage - feature for adoption. - For this , we already have a pure python library [itch](https://github.com/bbalouki/itch) so we want this to become a pure binding of the current library. But since it already has been published on pypi , we don't know the best action to take you decide. - Next is the itch message documentation, the current c++ version doe not docoment each message, use the pytohn version to read each message documenation and write it on the C++ using doxygen style (///) and (///<) -- **Apache Arrow / Parquet export** — turn a feed into columnar tables that drop - straight into pandas, Polars, DuckDB, and Spark pipelines. -- **CSV and streaming sinks** — simple, universal output for quick inspection and - legacy tooling. -- **`itch-tool` CLI** — a batteries-included command-line tool to inspect, filter, - convert (e.g. ITCH to Parquet), and produce message-type statistics/histograms - without writing any code. - ---- - -## Phase 5 — Simulation & ecosystem - -_Goal: close the loop for backtesting and become a maintained, depended-on package._ - -- **Replay engine with timestamp pacing** — drive consumers at original or scaled - wall-clock speed for realistic backtesting and system simulation. -- **ITCH encoder / writer** — synthesize valid ITCH streams for testing, scenario - generation, and golden fixtures (also unblocks much of the testing work in - [IMPROVEMENTS.md](IMPROVEMENTS.md) 6). -- **Multi-venue and multi-version support** — generalize via policy templates to - NASDAQ BX/PSX, older ITCH 4.1, and other ITCH-like venue feeds, so one codebase - covers a firm's whole equity data estate. -- **Packaging and project maturity** — publish to vcpkg and Conan, define a - versioned ABI/compatibility policy, and maintain a `CHANGELOG.md` and - `CONTRIBUTING.md` so the project is safe to depend on. - ---- - -## Non-goals - -To stay focused and best-in-class at what it does, ITCHCPP deliberately will not -become: - -- An order-management or execution-management system (OMS/EMS). -- An exchange matching engine. -- A strategy/alpha research framework or backtester proper (it _feeds_ those; it - is not one). - -Staying a superb data layer — ingest, reconstruct, analyze, export — is the way to -be used everywhere rather than competing with the systems built on top of it. From 9962c098260d906cb143bbbcab813c355374a3f5 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:16 +0100 Subject: [PATCH 040/144] docs: remove IMPROVEMENTS.md --- IMPROVEMENTS.md | 330 ------------------------------------------------ 1 file changed, 330 deletions(-) delete mode 100644 IMPROVEMENTS.md diff --git a/IMPROVEMENTS.md b/IMPROVEMENTS.md deleted file mode 100644 index b8db94c..0000000 --- a/IMPROVEMENTS.md +++ /dev/null @@ -1,330 +0,0 @@ -# IMPROVEMENTS - -This document is a prioritized audit of the current ITCHCPP codebase. It exists -to chart the path from a capable single-protocol file parser to infrastructure -that can credibly claim to be "one of the best in quantitative finance" and be -"usable everywhere in finance." - -Every finding below is tied to a specific location in the current source so it -can be acted on directly. Items are tagged by priority: - -- **P0** — correctness, safety, or a claim/reality gap that undermines trust. Fix first. -- **P1** — meaningful quality, performance, or process gaps. Fix soon. -- **P2** — polish and consistency. Fix opportunistically (Boy Scout Rule). - -New capabilities (transport, analytics, bindings, etc.) live in -[FEATURES.md](FEATURES.md). This file is about making what already exists correct, -fast, honest, and maintainable. - ---- - -## 1. Correctness & specification compliance - -### 1.1 Per-message length is never validated against the expected message size — P0 - -**Problem.** The framing loop reads the 2-byte length prefix, checks only that the -declared payload fits in the buffer, then dispatches on the type byte -([src/parser.cpp:291-322](src/parser.cpp)). The per-type `unpack_*` functions read -a fixed number of bytes based on the struct layout, never cross-checked against the -declared `length`. A message whose `length` is shorter than its type implies (a -corrupt or maliciously crafted frame) still dispatches, and the unpacker reads -fields belonging to the _next_ frame. - -**Impact.** Silent misparsing of adjacent data with no error raised. In a finance -context this is the worst failure mode: wrong numbers that look plausible. It is -also a robustness hole for untrusted input. - -**Fix.** Maintain a compile-time table of expected payload size per message type -and validate `length` against it before unpacking. On mismatch, route to the error -policy (see 1.2) rather than proceeding. This pairs naturally with the dispatch -rewrite in 2.1. - -### 1.2 The library prints to `std::cerr` on unknown message types — P0 - -**Problem.** When an unknown type byte is encountered the parser writes to -`std::cerr` ([src/parser.cpp:318](src/parser.cpp)). - -**Impact.** A library must never write to a global stream on a caller's behalf: it -corrupts the host application's output, is not thread-safe, and cannot be silenced. -It also violates the project's own house rule favoring `std::print`/`std::format` -over stream I/O. - -**Fix.** Replace with an explicit error policy: an optional error callback, an -accumulating diagnostics counter, or a `std::expected` return on the typed-parse -path. Default behavior should be silent-skip-and-count, configurable by the caller. - -### 1.3 `swap_bytes` relies on union type-punning — P0 - -**Problem.** Byte swapping is implemented by writing one union member and reading -another ([include/itch/parser.hpp:150-161](include/itch/parser.hpp)). Reading an -inactive union member is undefined behavior in C++ (it is only well-defined in C). - -**Impact.** Works today by compiler grace, but it is UB on the parser hot path — -exactly the code that must be bulletproof. Aggressive optimizers are within their -rights to miscompile it. - -**Fix.** Use `std::byteswap` (C++23, already a target standard) with a `std::endian` -check; fall back to `std::bit_cast` or compiler intrinsics -(`__builtin_bswap*` / `_byteswap_*`) for C++20. - ---- - -## 2. Performance — the headline gap versus the README - -The README promises "multi-gigabyte-per-second" throughput and "zero-overhead -deserialization." The implementation does not currently match that framing. These -items close the gap, or correct the claims. - -### 2.1 Dispatch uses `std::map>` — P0 - -**Problem.** Message dispatch is a `std::map>` -([include/itch/parser.hpp:133-134](include/itch/parser.hpp), -[src/parser.cpp:250-322](src/parser.cpp)). Every single message pays for a -red-black-tree lookup (pointer chasing, O(log n)) _plus_ a `std::function` -type-erased call (indirect call, possible heap, no inlining). - -**Impact.** This is the single biggest divergence from the "gigabytes/second, -zero-overhead" promise. On a feed of hundreds of millions of messages per day this -dominates the cost. - -**Fix.** Dispatch on the type byte through a 256-entry flat function-pointer table -or a `switch`. Type bytes are dense ASCII, so a flat array is cache-friendly and -branch-predictable, and lets the compiler inline each unpacker. Combine with 1.1's -size table so dispatch and validation share one lookup. - -### 2.2 The "zero-copy / zero-overhead deserialization" claim is inaccurate — P0 - -**Problem.** The README's architecture section states the parser reads messages -"without performing heavy data conversions" and copies bytes "skipping the usual -per-field decoding steps" ([README.md:83-85](README.md)). The actual code does -field-by-field `memcpy` plus per-field byte swap into separate host-order structs -([src/parser.cpp:69-263](src/parser.cpp)). That is per-field decoding, not -zero-copy. - -**Impact.** Overstated claims are a credibility risk with an infrastructure -audience that will read the source. - -**Fix.** Either (a) implement a genuine zero-copy overlay API that returns typed -views over the raw buffer with lazy, on-access endianness conversion, or -(b) correct the wording to describe what it does: a fast, allocation-free, -single-pass field decoder. Option (a) is the stronger product; see -[FEATURES.md](FEATURES.md). - -### 2.3 The order book is allocation-heavy — P1 - -**Problem.** Each resting order is a `std::shared_ptr` stored in a -`std::list`, with price levels held in `std::map` -([include/itch/order_book.hpp:25-61](include/itch/order_book.hpp), -[src/order_book.cpp](src/order_book.cpp)). That is an atomic refcount and a heap -allocation per order, plus a node allocation per list entry and per map node. - -**Impact.** This directly contradicts the zero-allocation positioning and makes the -book the bottleneck for any real reconstruction workload (book rebuilds touch every -add/cancel/execute/replace message). - -**Fix.** Move to an object pool / arena for orders, intrusive FIFO lists at each -price level, and flat (vector-backed) price ladders with O(1) order lookup by -reference number. Tracked in detail under the Phase 2 book engine in -[FEATURES.md](FEATURES.md). - -### 2.4 No published, reproducible benchmark numbers — P1 - -**Problem.** A Google Benchmark suite exists -([benchmarks/parser_bench.cpp](benchmarks/parser_bench.cpp)) but the README quotes -no concrete results, only aspirational ranges. - -**Impact.** "High performance" is unverifiable, and there is no regression signal -when 2.1–2.3 land. - -**Fix.** Publish a results table (hardware, compiler, dataset, msgs/sec, GB/s) and -wire the benchmark into CI to catch regressions. - ---- - -## 3. API modernization - -### 3.1 Replace `(const char*, size_t)` with `std::span` — P1 - -**Problem.** The buffer APIs take raw pointer/size pairs -([include/itch/parser.hpp:61-91](include/itch/parser.hpp)). - -**Fix.** Accept `std::span`. It carries the size, prevents -pointer/length desync, and is the modern idiom the project's standards call for. -Keep the pointer/size overloads as thin shims for C interop. - -### 3.2 Offer a non-throwing `std::expected` parse path — P1 - -**Problem.** Errors are reported only via exceptions (`std::runtime_error` on -truncation, [src/parser.cpp:296-309](src/parser.cpp)). - -**Impact.** Exceptions on the hot path are awkward for latency-sensitive callers, -and the project's standards explicitly suggest `std::expected` for predictable -failure paths. - -**Fix.** Provide a `std::expected`-based entry point alongside the -throwing wrappers. - -### 3.3 Introduce a typed fixed-point `Price` — P1 - -**Problem.** Prices are exposed as raw `uint32_t` and the caller is expected to -divide by `PRICE_DIVISOR` (10000) themselves -([include/itch/messages.hpp:327-329](include/itch/messages.hpp)), with the further -trap that MWCB decline levels use 8 decimals, not 4. - -**Impact.** Easy to misuse; mixing the two scales silently produces wrong prices. - -**Fix.** Add a strong `Price` type encoding scale, with explicit conversions to -`double`/decimal. Make the 4-vs-8 decimal distinction unrepresentable as a bug. - -### 3.4 Provide timestamp-to-wall-clock helpers — P1 - -**Problem.** Timestamps are exposed as raw nanoseconds-past-midnight `uint64_t` -with no utilities to combine them with a session date. - -**Fix.** Add helpers that convert to `std::chrono` time points given a session -date, plus formatting helpers. - ---- - -## 4. Adherence to the project's own C++ standards - -The repository ships a detailed [CLAUDE.md](CLAUDE.md) of C++ standards. Several -are violated by the current code. - -### 4.1 Symbols shorter than three characters — P2 - -**Problem.** The standard requires every symbol be at least three characters, yet -the code uses `val`, `src`, `dst`, `arr`, `sv`, single-letter loop indices `i`, -and template parameter `T` (e.g. -[include/itch/parser.hpp:150-161](include/itch/parser.hpp), -[include/itch/indicators.hpp:9](include/itch/indicators.hpp)). - -**Fix.** Rename to descriptive identifiers during the relevant refactors. - -### 4.2 Stream I/O instead of `std::print`/`std::format` — P2 - -**Problem.** The standard mandates `std::print`/`std::println`/`std::format`, but -output uses `std::cout`/`std::cerr` and iostream manipulators -([src/order_book.cpp:29-68](src/order_book.cpp), [src/parser.cpp:318](src/parser.cpp)). - -**Fix.** Migrate output paths to `std::format`/`std::print` (overlaps with 1.2). - -### 4.3 `std::map` globals defined in a header — P2 - -**Problem.** [include/itch/indicators.hpp](include/itch/indicators.hpp) defines -several non-trivially-constructed `std::map` objects at namespace scope in a header. -Each translation unit that includes it pays dynamic-initialization cost, and the -maps risk duplication. - -**Fix.** Replace with `constexpr` frozen lookups (sorted `std::array` of pairs with -binary search, or a compile-time map type) or functions returning `string_view`, -keeping the header dependency-light. - ---- - -## 5. Build, CI, and repository hygiene - -### 5.1 Large binary artifacts committed to the repository — P1 - -**Problem.** The repo tracks a full uncompressed NASDAQ ITCH sample -(`01302020.NASDAQ_ITCH50`), its `.gz` counterpart, and a generated coverage report -(`docs/coverage.html`). - -**Impact.** Bloated clone size, slow history, and a generated artifact under source -control. The data files in particular are large binaries that do not belong in git. - -**Fix.** Remove from history (or migrate to Git LFS / an external download script), -add them to `.gitignore`, and generate coverage into the build tree only. - -### 5.2 Inconsistent minimum CMake version — P2 - -**Problem.** The required CMake version disagrees across the project: README says -3.20, [cmake/Helpers.cmake:1](cmake/Helpers.cmake) says 3.23, and -[CLAUDE.md](CLAUDE.md) asks for >= 3.25. - -**Fix.** Standardize on a single minimum (>= 3.25 enables the presets workflow the -standards call for) and align every `cmake_minimum_required` and the docs. - -### 5.3 CI does not run on pull requests, and skips its own quality gates — P1 - -**Problem.** The workflow triggers only on `push` to `main` -([.github/workflows/build.yml:3-5](.github/workflows/build.yml)). It builds and -tests, but never runs clang-tidy, a clang-format check, sanitizers, or coverage — -even though [cmake/Helpers.cmake](cmake/Helpers.cmake) provides all of them. - -**Impact.** Regressions and style/lint violations can merge unreviewed by CI; -PRs get no signal at all. - -**Fix.** Add a `pull_request` trigger and dedicated jobs for clang-format check, -clang-tidy, an ASAN/UBSAN test run, and coverage reporting. - -### 5.4 No fuzz target for the parser — P1 - -**Problem.** The project standards mandate fuzz testing for parsers on untrusted -input, but no fuzz target exists. - -**Impact.** The framing/dispatch path (especially before 1.1 lands) is the textbook -candidate for crashes and overreads on malformed input. - -**Fix.** Add a libFuzzer/AFL++ target that feeds arbitrary bytes through `parse`, -and run it in CI on a time budget. - -### 5.5 Typos in public CMake helper names — P2 - -**Problem.** Helper functions are misspelled: `anable_address_sanitizer` and -`build_documenation` ([cmake/Helpers.cmake:136,206](cmake/Helpers.cmake)). - -**Impact.** These are part of the build's public surface; renaming later is a -breaking change for anyone who depends on them. - -**Fix.** Rename to `enable_address_sanitizer` / `build_documentation` now, while the -blast radius is small. - ---- - -## 6. Testing - -### 6.1 Add golden / conformance tests against the official sample — P1 - -**Problem.** There is no end-to-end conformance check that a known input produces -known output. - -**Fix.** Parse a committed-or-fetched official sample and assert message counts per -type and spot-checked field values against a golden fixture. - -### 6.2 Broaden edge-case coverage — P1 - -**Problem.** Current tests should be extended to cover the failure and boundary -modes that matter for a parser and a book. - -**Fix.** Add cases for truncated buffers, zero-length frames, unknown type bytes, -endianness round-trips, and the full order-book lifecycle (add, execute, -execute-with-price, cancel, delete, replace, and crossed-book scenarios). - ---- - -## Priority summary - -| ID | Item | Priority | Area | -| --- | ------------------------------------------------- | -------- | ---------------- | -| 1.1 | Validate message length vs. expected size | P0 | Correctness | -| 1.2 | Remove `std::cerr` from the library | P0 | Correctness | -| 1.3 | Replace union type-punning byte swap | P0 | Correctness (UB) | -| 2.1 | Replace `std::map` + `std::function` dispatch | P0 | Performance | -| 2.2 | Fix or implement the zero-copy claim | P0 | Honesty | -| 2.3 | Allocation-free order book internals | P1 | Performance | -| 2.4 | Publish reproducible benchmarks | P1 | Performance | -| 3.1 | `std::span` buffer API | P1 | API | -| 3.2 | `std::expected` parse path | P1 | API | -| 3.3 | Typed fixed-point `Price` | P1 | API | -| 3.4 | Timestamp-to-wall-clock helpers | P1 | API | -| 5.1 | Remove committed large binaries / coverage report | P1 | Repo hygiene | -| 5.3 | CI on PRs + lint/sanitizer/coverage gates | P1 | CI | -| 5.4 | Add a parser fuzz target | P1 | Testing | -| 6.1 | Golden / conformance tests | P1 | Testing | -| 6.2 | Edge-case coverage | P1 | Testing | -| 4.1 | Symbol length >= 3 | P2 | Standards | -| 4.2 | `std::print`/`std::format` output | P2 | Standards | -| 4.3 | Frozen/`constexpr` indicator lookups | P2 | Standards | -| 5.2 | Single CMake minimum version | P2 | Build | -| 5.5 | Fix CMake helper name typos | P2 | Build | From c2f678478442ec8f14346e8186910bcde48706ef Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:20 +0100 Subject: [PATCH 041/144] docs: fix broken CLAUDE/CHANGELOG links in CONTRIBUTING --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 326cbc7..63a2677 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,7 +34,7 @@ Optional features are off by default and enabled with: ## Code style -- Follow the rules in [CLAUDE.md](CLAUDE.md): `snake_case` functions/variables, +- Follow the rules in [CLAUDE](CLAUDE.md): `snake_case` functions/variables, `PascalCase` types, `m_` private members, `SCREAMING_SNAKE_CASE` constants, symbols at least three characters, trailing return types, brace initialization, and `std::print`/`std::format` for output. @@ -56,7 +56,7 @@ Optional features are off by default and enabled with: Python-binding job. - Keep the public API surface minimal; mark deprecations with `[[deprecated("reason")]]` and provide a migration path. -- Update [CHANGELOG.md](CHANGELOG.md) under `[Unreleased]` and follow the Boy +- Update [CHANGELOG](CHANGELOG.md) under `[Unreleased]` and follow the Boy Scout Rule. ## Versioning and compatibility policy From 59dd1807d6740fdc5e6d2b57bef822f698bf3459 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:27 +0100 Subject: [PATCH 042/144] docs: update README for new itchcpp package layout --- README.md | 81 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 45 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 280cc2c..f64d15c 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ A modern, high-performance C++20 library for parsing NASDAQ TotalView-ITCH 5.0 protocol data feeds. This parser is designed for maximum speed, minimal memory overhead, and type safety, making it ideal for latency-sensitive financial applications, market data analysis, and quantitative research. -📖 **API documentation:** (generated with Doxygen, published on every push to `main`). +📖 [**API documentation:**](https://bbalouki.github.io/itchcpp/). --- @@ -66,9 +66,9 @@ The design of this ITCH parser is guided by three principles: ## Key Features -- **Real Feed Ingestion**: Consume ITCH as it actually arrives — MoldUDP64 UDP +- **Real Feed Ingestion**: Consume ITCH as it actually arrives, MoldUDP64 UDP multicast framing, SoupBinTCP (Glimpse/recovery) framing, and `.pcap`/`.pcapng` - captures — with per-session sequence tracking and gap detection. No libpcap + captures, with per-session sequence tracking and gap detection. No libpcap dependency. See [Feed Ingestion](#feed-ingestion-transport). - **Full-Market Book Engine**: Reconstruct every symbol on the feed in one pass with an allocation-light L3 order book (object pool, intrusive FIFO levels, flat @@ -77,7 +77,7 @@ The design of this ITCH parser is guided by three principles: - **Zero-Copy Overlay API**: Inspect raw frames through lazy typed views that convert only the fields you read, for hot paths that touch a few fields per message. -- **Built-in Analytics**: Header-only microstructure layer — OHLCV bar builders +- **Built-in Analytics**: Header-only microstructure layer, OHLCV bar builders (time/tick/volume clocks), VWAP/TWAP, spread, queue imbalance, order-flow imbalance, NOII surfacing, and auction reconstruction. See [Analytics](#analytics). - **Interoperability**: CSV and Arrow/Parquet export, a batteries-included @@ -109,7 +109,7 @@ The parser is a fast, allocation-free, single-pass field decoder. Each message t Dispatch is driven by a compile-time table keyed on the one-byte message type, so selecting the right decoder is a single flat-array lookup followed by a direct, inlinable call. There is no red-black-tree lookup and no type-erased `std::function` indirection. The frame length declared on the wire is validated against the size the message type requires before any field is read, so a truncated or malformed frame can never cause the decoder to read into an adjacent message. -This is genuine single-pass decoding rather than a zero-copy overlay: the bytes are converted eagerly into host-order fields. A lazy zero-copy view API (typed overlays over the raw buffer with on-access endianness conversion) is tracked as future work in [FEATURES.md](FEATURES.md). +This is genuine single-pass decoding rather than a zero-copy overlay: the bytes are converted eagerly into host-order fields. ### Type-Safe Message Handling @@ -472,12 +472,12 @@ delivered inside transport framing. The `itch::transport` module decodes the framing that actually arrives on the wire and on disk, then feeds the existing parser. Everything is implemented in-house with **no libpcap dependency**. -| Decoder | Header | Purpose | -| ------- | ------ | ------- | -| `MoldUdp64Decoder` | `itch/transport/moldudp64.hpp` | UDP multicast framing for live dissemination. | -| `SoupBinDecoder` | `itch/transport/soupbintcp.hpp` | TCP framing for Glimpse snapshots and recovery/replay. | -| `PcapReader` | `itch/transport/pcap.hpp` | Replay a feed from a `.pcap`/`.pcapng` capture file. | -| `SequenceTracker` | `itch/transport/sequencing.hpp` | Per-session sequence tracking, gap detection, recovery hooks. | +| Decoder | Header | Purpose | +| ------------------ | ------------------------------- | ------------------------------------------------------------- | +| `MoldUdp64Decoder` | `itch/transport/moldudp64.hpp` | UDP multicast framing for live dissemination. | +| `SoupBinDecoder` | `itch/transport/soupbintcp.hpp` | TCP framing for Glimpse snapshots and recovery/replay. | +| `PcapReader` | `itch/transport/pcap.hpp` | Replay a feed from a `.pcap`/`.pcapng` capture file. | +| `SequenceTracker` | `itch/transport/sequencing.hpp` | Per-session sequence tracking, gap detection, recovery hooks. | ```cpp #include "itch/transport/pcap.hpp" @@ -546,13 +546,13 @@ The header-only `itch::analytics` layer computes the metrics quants ask for, directly off the trade tape and book so every downstream team does not reimplement them. -| Component | Header | Provides | -| --------- | ------ | -------- | -| `BarBuilder` | `analytics/bars.hpp` | OHLCV bars over `TimeClock`, `TickClock`, `VolumeClock`. | -| `Vwap`, `Twap` | `analytics/vwap.hpp` | Running/interval volume- and time-weighted average price. | +| Component | Header | Provides | +| ------------------------ | ------------------------------ | ------------------------------------------------------------------- | +| `BarBuilder` | `analytics/bars.hpp` | OHLCV bars over `TimeClock`, `TickClock`, `VolumeClock`. | +| `Vwap`, `Twap` | `analytics/vwap.hpp` | Running/interval volume- and time-weighted average price. | | spread / mid / imbalance | `analytics/microstructure.hpp` | Spread, mid, depth-at-level, queue imbalance, order-flow imbalance. | -| `ImbalanceInfo` | `analytics/imbalance.hpp` | Decoded NOII (`I`) imbalance data. | -| `AuctionTracker` | `analytics/auctions.hpp` | Opening/closing/halt/IPO cross reconstruction. | +| `ImbalanceInfo` | `analytics/imbalance.hpp` | Decoded NOII (`I`) imbalance data. | +| `AuctionTracker` | `analytics/auctions.hpp` | Opening/closing/halt/IPO cross reconstruction. | ```cpp #include "itch/analytics/vwap.hpp" @@ -590,16 +590,25 @@ itch-tool filter data.itch --types AEP --out trades.csv itch-tool convert data.itch --out data.csv # ITCH -> CSV (-> Parquet w/ Arrow) ``` -**Python bindings** (`-DITCH_BUILD_PYTHON=ON`, pybind11). A native module that -mirrors the pure-Python [`itch`](https://github.com/bbalouki/itch) package's API -so it can act as a faster drop-in backend. See [python/README.md](python/README.md). +**Python bindings** (`-DITCH_BUILD_PYTHON=ON`, pybind11). The `itchcpp` package is +a faster, drop-in backend for the pure-Python +[`itch`](https://github.com/bbalouki/itch) package (PyPI: +[`itchfeed`](https://pypi.org/project/itchfeed/)). It mirrors that package's layout +(`itchcpp.messages`, `itchcpp.parser`, `itchcpp.indicators`) and semantics, the same +message classes with the same raw `bytes`/`int` attributes, the same `MessageParser` +(type filter, lazy iteration, `parse_file`/`parse_stream`/`parse_messages`), +`create_message`, and `decode`/`decode_price`/`to_bytes` helpers, so migrating only +changes the import root. See [itchcpp](python/README.md). ```python -import itchcpp -parser = itchcpp.MessageParser() -for message in parser.parse_file("01302020.NASDAQ_ITCH50"): - if isinstance(message, itchcpp.AddOrderMessage): - print(message.stock, message.decode_price("price"), message.shares) +from itchcpp.parser import MessageParser +from itchcpp.messages import AddOrderNoMPIAttributionMessage + +parser = MessageParser() # MessageParser(message_type=b"AFE") to filter types +with open("01302020.NASDAQ_ITCH50", "rb") as itch_file: + for message in parser.parse_file(itch_file): + if isinstance(message, AddOrderNoMPIAttributionMessage): + print(message.stock, message.decode_price("price"), message.shares) ``` ### Simulation & Ecosystem @@ -615,7 +624,7 @@ engine.replay(buffer, [](const itch::Message& msg) { /* paced by timestamps */ } ``` **Encoder / writer** (`itch/encoder.hpp`). Serialize any message back to valid -wire bytes, with a guaranteed `parse(encode(msg)) == msg` round-trip — used to +wire bytes, with a guaranteed `parse(encode(msg)) == msg` round-trip, used to synthesize streams and golden fixtures: ```cpp @@ -631,7 +640,7 @@ the dispatch machinery. **Packaging.** The library is consumable through vcpkg (manifest with optional `python` and `arrow` features) and Conan (`conanfile.py`). See -[CONTRIBUTING.md](CONTRIBUTING.md) for the build options and the versioning/ABI +[CONTRIBUTING](CONTRIBUTING.md) for the build options and the versioning/ABI compatibility policy. --- @@ -698,11 +707,11 @@ The parser is designed for high-throughput scenarios. Performance is heavily dep The numbers below were produced by [`benchmarks/parser_bench.cpp`](benchmarks/parser_bench.cpp) parsing from an in-memory buffer (file I/O excluded). They are reproducible with the setup described under [How to reproduce](#how-to-reproduce). -| Configuration | Throughput | -| ------------- | ---------- | -| `parse(..., callback)` — allocation-free callback path | **2.24 GiB/s** | -| `parse(...)` — collect all messages into a `std::vector` | 1.23 GiB/s | -| `parse(..., {'A','P','E','C','X'})` — filtered into a `std::vector` | 1.15 GiB/s | +| Configuration | Throughput | +| ------------------------------------------------------------------ | -------------- | +| `parse(..., callback)`, allocation-free callback path | **2.24 GiB/s** | +| `parse(...)`, collect all messages into a `std::vector` | 1.23 GiB/s | +| `parse(..., {'A','P','E','C','X'})`, filtered into a `std::vector` | 1.15 GiB/s | - **Hardware**: x86-64, 16 logical cores @ 1.70 GHz (L1d 32 KiB, L2 512 KiB, L3 4 MiB). - **Compiler**: Clang 22, `-DCMAKE_BUILD_TYPE=Release`, C++23. @@ -718,10 +727,10 @@ The throughput figures below were measured on a synthetic, churn-heavy stream Clang 22 `-DCMAKE_BUILD_TYPE=Release`, C++23; book-rebuild throughput on real data depends heavily on the depth and churn profile of the feed. -| Configuration | Throughput | -| ------------- | ---------- | -| `BookManager::process` — full multi-symbol L3 reconstruction | ~150 MiB/s | -| Eager `parse`, touching one field per message | ~5.4 GiB/s | +| Configuration | Throughput | +| --------------------------------------------------------------------- | -------------- | +| `BookManager::process`, full multi-symbol L3 reconstruction | ~150 MiB/s | +| Eager `parse`, touching one field per message | ~5.4 GiB/s | | Zero-copy `overlay::for_each_message`, touching one field per message | **~9.7 GiB/s** | The overlay is materially faster than the eager decoder when only a few fields are From 135dbe43a124055bbd78dfd092852898d8744e5e Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:30 +0100 Subject: [PATCH 043/144] style: use Doxygen /// comments in book_bench --- benchmarks/book_bench.cpp | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/benchmarks/book_bench.cpp b/benchmarks/book_bench.cpp index 02b19b9..10fbe08 100644 --- a/benchmarks/book_bench.cpp +++ b/benchmarks/book_bench.cpp @@ -1,16 +1,14 @@ -/** - * @file book_bench.cpp - * @brief Benchmarks for the Phase 2 book engine and the zero-copy overlay. - * - * Measures three things against an in-memory ITCH buffer (file I/O excluded): - * - BM_BookRebuild: full multi-symbol book reconstruction through BookManager. - * - BM_EagerTouch: eager parse touching one field per message (the baseline). - * - BM_OverlayTouch: lazy overlay framing touching the same one field, which - * should be cheaper because the other fields are never decoded. - * - * Usage: - * ./book_bench [google benchmark options] - */ +/// @file book_bench.cpp +/// @brief Benchmarks for the Phase 2 book engine and the zero-copy overlay. +/// +/// Measures three things against an in-memory ITCH buffer (file I/O excluded): +/// - BM_BookRebuild: full multi-symbol book reconstruction through BookManager. +/// - BM_EagerTouch: eager parse touching one field per message (the baseline). +/// - BM_OverlayTouch: lazy overlay framing touching the same one field, which +/// should be cheaper because the other fields are never decoded. +/// +/// Usage: +/// ./book_bench [google benchmark options] #include From b8004edf7008ce4d771a2ed9a49af9763b184cc5 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:33 +0100 Subject: [PATCH 044/144] style: use Doxygen /// comments in parser_bench --- benchmarks/parser_bench.cpp | 47 ++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/benchmarks/parser_bench.cpp b/benchmarks/parser_bench.cpp index 38b61ea..5221d4e 100644 --- a/benchmarks/parser_bench.cpp +++ b/benchmarks/parser_bench.cpp @@ -1,27 +1,26 @@ -/** - * @file parser_bench.cpp - * @brief Benchmarking suite for the ITCH message parser using Google Benchmark. - * - * This file defines a set of benchmarks to evaluate the performance of the - * ITCH message parser implemented in the Parser class. It uses the Google - * Benchmark library to measure execution time and throughput for different - * parsing strategies, including callback-based parsing, collecting all parsed - * messages, and filtering specific message types. - * - * The benchmark fixture `ParserBenchmark` is responsible for loading the ITCH - * data file into memory once per benchmark run, ensuring that file I/O does - * not skew the parsing performance measurements. - * - * Usage: - * ./parser_bench.exe [google benchmark options] - * - * Example: - * ./parser_bench.exe data/itch_data.bin --benchmark_filter=BM_ParseWithCallback - * - * Note: - * Ensure that the Google Benchmark library is properly linked during - * compilation. - */ + +/// @file parser_bench.cpp +/// @brief Benchmarking suite for the ITCH message parser using Google Benchmark. +/// +/// This file defines a set of benchmarks to evaluate the performance of the +/// ITCH message parser implemented in the Parser class. It uses the Google +/// Benchmark library to measure execution time and throughput for different +/// parsing strategies, including callback-based parsing, collecting all parsed +/// messages, and filtering specific message types. +/// +/// The benchmark fixture `ParserBenchmark` is responsible for loading the ITCH +/// data file into memory once per benchmark run, ensuring that file I/O does +/// not skew the parsing performance measurements. +/// +/// Usage: +/// ./parser_bench.exe [google benchmark options] +/// +/// Example: +/// ./parser_bench.exe data/itch_data.bin --benchmark_filter=BM_ParseWithCallback +/// +/// Note: +/// Ensure that the Google Benchmark library is properly linked during +/// compilation. #include From b1d6c308cd697bf06fb453fffd3e57dabe6bfa07 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:37 +0100 Subject: [PATCH 045/144] docs: trim trailing whitespace in messages.hpp --- include/itch/messages.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/itch/messages.hpp b/include/itch/messages.hpp index aa9285e..56abe64 100644 --- a/include/itch/messages.hpp +++ b/include/itch/messages.hpp @@ -439,7 +439,7 @@ struct DLCRMessage { /// - stock_locate: Locate code identifying the security /// - tracking_number: Nasdaq internal tracking number /// -/// for more details on each message type, see the +/// for more details on each message type, see the /// [documentation](https://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NQTVITCHspecification.pdf). /// /// @note From ab04f6d9812d916c715e89f0bcb437c5ab09d8ed Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:43 +0100 Subject: [PATCH 046/144] cmake: remove FetchDocsTheme script --- cmake/FetchDocsTheme.cmake | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 cmake/FetchDocsTheme.cmake diff --git a/cmake/FetchDocsTheme.cmake b/cmake/FetchDocsTheme.cmake deleted file mode 100644 index 85c15ac..0000000 --- a/cmake/FetchDocsTheme.cmake +++ /dev/null @@ -1,22 +0,0 @@ -# Clones the doxygen-awesome-css theme into the docs tree if it is absent, so the -# local `docs` target produces the same modern styling as the published site. The -# checkout is gitignored. Invoked by the `docs` custom target with -P. - -if(EXISTS "${THEME_DIR}/doxygen-awesome.css") - return() -endif() - -if(NOT GIT_EXECUTABLE) - message(WARNING "Git not found; docs will use the default Doxygen theme.") - return() -endif() - -message(STATUS "Fetching doxygen-awesome-css (${THEME_TAG}) into ${THEME_DIR} ...") -execute_process( - COMMAND "${GIT_EXECUTABLE}" clone --depth 1 --branch "${THEME_TAG}" - https://github.com/jothepro/doxygen-awesome-css.git "${THEME_DIR}" - RESULT_VARIABLE clone_result -) -if(NOT clone_result EQUAL 0) - message(WARNING "Could not fetch doxygen-awesome-css; docs will use the default theme.") -endif() From 9419aabc285e07beda34b90a3eb6916a71111365 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:46 +0100 Subject: [PATCH 047/144] cmake: pin doxygen-awesome-css version --- cmake/Versions.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/Versions.cmake b/cmake/Versions.cmake index 0eff013..00c5260 100644 --- a/cmake/Versions.cmake +++ b/cmake/Versions.cmake @@ -2,3 +2,4 @@ cmake_minimum_required(VERSION 3.25) set(GTEST_VERSION 1.17.0 CACHE STRING "") set(BENCHMARK_VERSION 1.9.4 CACHE STRING "") +set(DOXYGEN_AWESOME_VERSION 2.4.2 CACHE STRING "") From bdc0650f17798cbbda3d5951071cfd111aaf1ed3 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:49 +0100 Subject: [PATCH 048/144] cmake: fetch doxygen theme via FetchContent, use Doxyfile.in --- cmake/Helpers.cmake | 65 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 16 deletions(-) diff --git a/cmake/Helpers.cmake b/cmake/Helpers.cmake index 3a6fa97..75e74e6 100644 --- a/cmake/Helpers.cmake +++ b/cmake/Helpers.cmake @@ -145,25 +145,58 @@ function(build_documentation) "Found Doxygen Version " "${DOXYGEN_VERSION} " "at ${DOXYGEN_EXECUTABLE}" ) - find_package(Git QUIET) - # Run Doxygen on the committed docs/Doxyfile from the source root so its - # relative INPUT paths (README.md, include, src) resolve. The same - # Doxyfile is used by the GitHub Pages workflow, keeping local and - # published documentation identical. The modern theme is fetched first so - # local docs are styled like the published site. - add_custom_target( - docs - COMMAND ${CMAKE_COMMAND} - -DTHEME_DIR=${PROJECT_SOURCE_DIR}/docs/doxygen-awesome-css - -DTHEME_TAG=v2.3.4 - -DGIT_EXECUTABLE=${GIT_EXECUTABLE} - -P ${PROJECT_SOURCE_DIR}/cmake/FetchDocsTheme.cmake - COMMAND ${DOXYGEN_EXECUTABLE} "${PROJECT_SOURCE_DIR}/docs/Doxyfile" - WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + + # Fetch the doxygen-awesome-css theme. + include(FetchContent) + FetchContent_Declare( + doxygen-awesome-css + GIT_REPOSITORY https://github.com/jothepro/doxygen-awesome-css.git + GIT_TAG "v${DOXYGEN_AWESOME_VERSION}" + SOURCE_SUBDIR "do-not-add-subdir" + ) + FetchContent_MakeAvailable(doxygen-awesome-css) + set(DOXYGEN_AWESOME_DIR "${doxygen-awesome-css_SOURCE_DIR}") + + # Generate a header from the active Doxygen version and inject the theme scripts before + # . Generating against the real binary keeps the header in sync with whatever + # Doxygen produces, instead of committing a hand-written header that drifts per version. + set(DOXYGEN_DEFAULT_HEADER "${CMAKE_CURRENT_BINARY_DIR}/header.default.html") + set(DOXYGEN_HTML_HEADER "${CMAKE_CURRENT_BINARY_DIR}/header.html") + execute_process( + COMMAND "${DOXYGEN_EXECUTABLE}" -w html + "${DOXYGEN_DEFAULT_HEADER}" + "${CMAKE_CURRENT_BINARY_DIR}/footer.default.html" + "${CMAKE_CURRENT_BINARY_DIR}/style.default.css" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + ) + file(READ "${DOXYGEN_DEFAULT_HEADER}" DOXYGEN_HEADER_CONTENT) + string(CONCAT DOXYGEN_THEME_SCRIPTS + "\n" + "\n" + "\n" + "\n" + "\n" + ) + string(REPLACE "" "${DOXYGEN_THEME_SCRIPTS}" DOXYGEN_HEADER_CONTENT "${DOXYGEN_HEADER_CONTENT}") + file(WRITE "${DOXYGEN_HTML_HEADER}" "${DOXYGEN_HEADER_CONTENT}") + + configure_file( + "${PROJECT_SOURCE_DIR}/docs/Doxyfile.in" + "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" + @ONLY + ) + doxygen_add_docs( + docs + CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" COMMENT "Generating docs with Doxygen ..." - VERBATIM ) message(STATUS "Add 'docs' target") + message(STATUS "To build the documentation, run: cmake --build . --target docs") endif() endfunction() From cd2612946f00ea18bf1bf8e3af805fd3c418c796 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:53 +0100 Subject: [PATCH 049/144] docs: remove static Doxyfile --- docs/Doxyfile | 96 --------------------------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 docs/Doxyfile diff --git a/docs/Doxyfile b/docs/Doxyfile deleted file mode 100644 index 262df8e..0000000 --- a/docs/Doxyfile +++ /dev/null @@ -1,96 +0,0 @@ -# Doxygen configuration for ITCHCPP. -# -# Run from the repository root: `doxygen docs/Doxyfile` (the CMake `docs` target -# does this for you). Output is written to docs/html. The modern look comes from -# doxygen-awesome-css, referenced below; the GitHub Pages workflow clones it into -# docs/doxygen-awesome-css and generates a matching HTML header for the active -# Doxygen version. - -#--------------------------------------------------------------------------- -# Project -#--------------------------------------------------------------------------- -PROJECT_NAME = "ITCHCPP" -PROJECT_NUMBER = 1.6.0 -PROJECT_BRIEF = "High-Performance NASDAQ TotalView-ITCH 5.0 Parser & Market-Data Platform" -OUTPUT_DIRECTORY = docs -HTML_OUTPUT = html -CREATE_SUBDIRS = NO -FULL_PATH_NAMES = NO -JAVADOC_AUTOBRIEF = YES -QT_AUTOBRIEF = YES -TAB_SIZE = 4 -MARKDOWN_SUPPORT = YES -# GitHub-style heading anchors so the README's table-of-contents links resolve. -MARKDOWN_ID_STYLE = GITHUB -TOC_INCLUDE_HEADINGS = 4 -AUTOLINK_SUPPORT = YES -BUILTIN_STL_SUPPORT = YES - -#--------------------------------------------------------------------------- -# Build / extraction -#--------------------------------------------------------------------------- -EXTRACT_ALL = YES -EXTRACT_STATIC = YES -EXTRACT_LOCAL_CLASSES = YES -RECURSIVE = YES -SORT_MEMBER_DOCS = NO -QUIET = YES -WARN_IF_UNDOCUMENTED = NO - -#--------------------------------------------------------------------------- -# Input -#--------------------------------------------------------------------------- -INPUT = README.md \ - FEATURES.md \ - IMPROVEMENTS.md \ - CONTRIBUTING.md \ - CHANGELOG.md \ - python/README.md \ - include \ - src -FILE_PATTERNS = *.hpp *.h *.cpp *.md -USE_MDFILE_AS_MAINPAGE = README.md -EXCLUDE_PATTERNS = */build/* */build-*/* */.venv/* - -#--------------------------------------------------------------------------- -# Source browsing -#--------------------------------------------------------------------------- -SOURCE_BROWSER = YES -REFERENCED_BY_RELATION = YES -REFERENCES_RELATION = YES - -#--------------------------------------------------------------------------- -# Output formats -#--------------------------------------------------------------------------- -GENERATE_HTML = YES -GENERATE_LATEX = NO -GENERATE_XML = NO - -#--------------------------------------------------------------------------- -# HTML appearance - modern theme (doxygen-awesome-css) -#--------------------------------------------------------------------------- -# doxygen-awesome requires the light base color style and works best with a -# tree view sidebar. The extra stylesheets and JS files are provided by the -# cloned doxygen-awesome-css checkout; the dark-mode toggle, copy buttons, and -# interactive table of contents are wired up by the generated HTML header. -GENERATE_TREEVIEW = YES -DISABLE_INDEX = NO -FULL_SIDEBAR = NO -HTML_COLORSTYLE = LIGHT -HTML_DYNAMIC_SECTIONS = YES -HTML_COPY_CLIPBOARD = NO -HTML_EXTRA_STYLESHEET = docs/doxygen-awesome-css/doxygen-awesome.css \ - docs/doxygen-awesome-css/doxygen-awesome-sidebar-only.css \ - docs/doxygen-awesome-css/doxygen-awesome-sidebar-only-darkmode-toggle.css -HTML_EXTRA_FILES = docs/doxygen-awesome-css/doxygen-awesome-darkmode-toggle.js \ - docs/doxygen-awesome-css/doxygen-awesome-fragment-copy-button.js \ - docs/doxygen-awesome-css/doxygen-awesome-paragraph-link.js \ - docs/doxygen-awesome-css/doxygen-awesome-interactive-toc.js - -#--------------------------------------------------------------------------- -# Diagrams (graphviz; optional, enabled when dot is available) -#--------------------------------------------------------------------------- -HAVE_DOT = NO -CLASS_GRAPH = YES -COLLABORATION_GRAPH = NO -UML_LOOK = YES From b2277a6aa4e86e2a933a5274767e7e595687586e Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:56 +0100 Subject: [PATCH 050/144] docs: add CMake-configured Doxyfile template --- docs/Doxyfile.in | 98 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 docs/Doxyfile.in diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in new file mode 100644 index 0000000..15f7522 --- /dev/null +++ b/docs/Doxyfile.in @@ -0,0 +1,98 @@ +# Doxygen configuration template for ITCHCPP. +# +# This is a CMake-configured template: build_documentation() in cmake/Helpers.cmake +# substitutes the @VARS@ below (the fetched doxygen-awesome-css theme directory, the +# generated HTML header, the project version, and absolute source paths) and adds a +# `docs` target. Build it with: +# +# cmake -S . -B build -DITCH_BUILD_DOCUMENTATION=ON +# cmake --build build --target docs +# +# Output is written to docs/html. The theme is fetched at configure time, so no +# theme files need to live in the repository. + +#--------------------------------------------------------------------------- +# Project +#--------------------------------------------------------------------------- +PROJECT_NAME = "ITCHCPP" +PROJECT_NUMBER = @PROJECT_VERSION@ +PROJECT_BRIEF = "High-Performance NASDAQ TotalView-ITCH 5.0 Parser & Market-Data Platform" +OUTPUT_DIRECTORY = @PROJECT_SOURCE_DIR@/docs +HTML_OUTPUT = html +CREATE_SUBDIRS = NO +FULL_PATH_NAMES = NO +JAVADOC_AUTOBRIEF = YES +QT_AUTOBRIEF = YES +TAB_SIZE = 4 +MARKDOWN_SUPPORT = YES +# GitHub-style heading anchors so the README's table-of-contents links resolve. +MARKDOWN_ID_STYLE = GITHUB +TOC_INCLUDE_HEADINGS = 4 +AUTOLINK_SUPPORT = YES +BUILTIN_STL_SUPPORT = YES + +#--------------------------------------------------------------------------- +# Build / extraction +#--------------------------------------------------------------------------- +EXTRACT_ALL = YES +EXTRACT_STATIC = YES +EXTRACT_LOCAL_CLASSES = YES +RECURSIVE = YES +SORT_MEMBER_DOCS = NO +QUIET = YES +WARN_IF_UNDOCUMENTED = NO + +#--------------------------------------------------------------------------- +# Input (absolute paths so the build works from any directory) +#--------------------------------------------------------------------------- +INPUT = @PROJECT_SOURCE_DIR@/README.md \ + @PROJECT_SOURCE_DIR@/python/README.md \ + @PROJECT_SOURCE_DIR@/include \ + @PROJECT_SOURCE_DIR@/src +FILE_PATTERNS = *.hpp *.h *.cpp *.md +USE_MDFILE_AS_MAINPAGE = @PROJECT_SOURCE_DIR@/README.md +EXCLUDE_PATTERNS = */build/* */build-*/* */.venv/* + +#--------------------------------------------------------------------------- +# Source browsing +#--------------------------------------------------------------------------- +SOURCE_BROWSER = YES +REFERENCED_BY_RELATION = YES +REFERENCES_RELATION = YES + +#--------------------------------------------------------------------------- +# Output formats +#--------------------------------------------------------------------------- +GENERATE_HTML = YES +GENERATE_LATEX = NO +GENERATE_XML = NO + +#--------------------------------------------------------------------------- +# HTML appearance - modern theme (doxygen-awesome-css) +#--------------------------------------------------------------------------- +# The theme directory (@DOXYGEN_AWESOME_DIR@) and the generated header +# (@DOXYGEN_HTML_HEADER@) are produced by build_documentation() at configure time. +GENERATE_TREEVIEW = YES +DISABLE_INDEX = NO +FULL_SIDEBAR = NO +HTML_COLORSTYLE = LIGHT +HTML_DYNAMIC_SECTIONS = YES +HTML_COPY_CLIPBOARD = NO +HTML_HEADER = @DOXYGEN_HTML_HEADER@ +HTML_EXTRA_STYLESHEET = @DOXYGEN_AWESOME_DIR@/doxygen-awesome.css \ + @DOXYGEN_AWESOME_DIR@/doxygen-awesome-sidebar-only.css \ + @DOXYGEN_AWESOME_DIR@/doxygen-awesome-sidebar-only-darkmode-toggle.css +HTML_EXTRA_FILES = @DOXYGEN_AWESOME_DIR@/doxygen-awesome-darkmode-toggle.js \ + @DOXYGEN_AWESOME_DIR@/doxygen-awesome-fragment-copy-button.js \ + @DOXYGEN_AWESOME_DIR@/doxygen-awesome-paragraph-link.js \ + @DOXYGEN_AWESOME_DIR@/doxygen-awesome-interactive-toc.js + +#--------------------------------------------------------------------------- +# Diagrams (graphviz) +#--------------------------------------------------------------------------- +HAVE_DOT = YES +DOT_IMAGE_FORMAT = svg +DOT_TRANSPARENT = YES +CLASS_GRAPH = YES +COLLABORATION_GRAPH = NO +UML_LOOK = YES From 2328e5fa9f1bb1f53d2d88e19d52e425617c1224 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:39:59 +0100 Subject: [PATCH 051/144] docs: remove static awesome-head.html --- docs/awesome-head.html | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 docs/awesome-head.html diff --git a/docs/awesome-head.html b/docs/awesome-head.html deleted file mode 100644 index 8e55835..0000000 --- a/docs/awesome-head.html +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - From 0cadcd86c0afacf242d0dafcbe4b63da934737de Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:05 +0100 Subject: [PATCH 052/144] style: wrap long line in conanfile.py --- conanfile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/conanfile.py b/conanfile.py index 84b1578..dd2e7fa 100644 --- a/conanfile.py +++ b/conanfile.py @@ -34,7 +34,9 @@ class ItchConan(ConanFile): ) def set_version(self): - self.version = load(self, os.path.join(self.recipe_folder, "VERSION.txt")).strip() + self.version = load( + self, os.path.join(self.recipe_folder, "VERSION.txt") + ).strip() def requirements(self): if self.options.with_arrow: From 46e264557914b8bae7e1d2221b3926c7b8be84d2 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:12 +0100 Subject: [PATCH 053/144] packaging: remove root vcpkg.json --- vcpkg.json | 26 -------------------------- 1 file changed, 26 deletions(-) delete mode 100644 vcpkg.json diff --git a/vcpkg.json b/vcpkg.json deleted file mode 100644 index f268b88..0000000 --- a/vcpkg.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "dependencies": [ - "gtest", - "benchmark" - ], - "features": { - "python": { - "description": "Build the pybind11 Python bindings", - "dependencies": [ - "pybind11" - ] - }, - "arrow": { - "description": "Enable Apache Arrow / Parquet columnar export", - "dependencies": [ - { - "name": "arrow", - "features": [ - "parquet" - ] - } - ] - } - }, - "builtin-baseline": "5bf0c55239da398b8c6f450818c9e28d36bf9966" -} From 6901ed6975b425f2389c48ac5c8a33012c276ad1 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:16 +0100 Subject: [PATCH 054/144] packaging: add root pyproject.toml for itchcpp wheel --- pyproject.toml | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b9fb92c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,58 @@ +# Python packaging for the `itchcpp` wheel. + +[build-system] +requires = ["scikit-build-core>=0.8", "pybind11>=2.11"] +build-backend = "scikit_build_core.build" + +[project] +name = "itchcpp" +dynamic = ["version"] +description = "Native C++ NASDAQ TotalView-ITCH 5.0 parser, book engine, and analytics: a fast, drop-in backend for the pure-Python `itch` (itchfeed) package." +readme = "python/README.md" +requires-python = ">=3.9" +license = { text = "MIT" } +authors = [{ name = "Bertin Balouki Simyeli" }] +keywords = ["nasdaq", "itch", "market-data", "order-book", "hft"] +classifiers = [ + "Programming Language :: C++", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Office/Business :: Financial :: Investment", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/bbalouki/itchcpp" +Documentation = "https://bbalouki.github.io/itchcpp/" + +[tool.scikit-build] +cmake.source-dir = "." +cmake.args = ["-DITCH_BUILD_PYTHON=ON", "-DITCH_CXX_STANDARD=20"] +cmake.define = { "CMAKE_VS_GLOBALS" = "IgnoreWarnIntDirInTempDetected=true" } +wheel.packages = ["python/itchcpp"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "VERSION.txt" +regex = "(?P.+)" + +[tool.cibuildwheel] +build = "cp39-* cp310-* cp311-* cp312-* cp313-*" +skip = "*-win32 *_i686 *-musllinux_*" +build-frontend = "build" + +manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" +test-requires = "pytest" +test-command = "pytest {project}/python/tests" + +[tool.cibuildwheel.macos.environment] +# C++20 with libc++ needs a recent deployment target. +MACOSX_DEPLOYMENT_TARGET = "11.0" From 2479c1284320b4d6f07d9212c6964b895d0554a9 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:19 +0100 Subject: [PATCH 055/144] packaging: remove old python/pyproject.toml, moved to root --- python/pyproject.toml | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 python/pyproject.toml diff --git a/python/pyproject.toml b/python/pyproject.toml deleted file mode 100644 index a03b5a3..0000000 --- a/python/pyproject.toml +++ /dev/null @@ -1,28 +0,0 @@ -[build-system] -requires = ["scikit-build-core>=0.8", "pybind11>=2.11"] -build-backend = "scikit_build_core.build" - -[project] -name = "itchcpp" -version = "1.5.0" -description = "Native C++ NASDAQ TotalView-ITCH 5.0 parser, book engine, and analytics (a fast backend for the pure-Python `itch` package)." -readme = "README.md" -requires-python = ">=3.9" -license = { text = "MIT" } -authors = [{ name = "Bertin Balouki Simyeli" }] -keywords = ["nasdaq", "itch", "market-data", "order-book", "hft"] -classifiers = [ - "Programming Language :: C++", - "Programming Language :: Python :: 3", - "Topic :: Office/Business :: Financial :: Investment", - "License :: OSI Approved :: MIT License", -] - -[project.urls] -Homepage = "https://github.com/bbalouki/itchcpp" - -[tool.scikit-build] -# The bindings live under python/ but compile the library from the repo root. -cmake.source-dir = ".." -cmake.args = ["-DITCH_BUILD_PYTHON=ON"] -wheel.packages = [] From 11c00d47fb45d223fa923e0051c3f4f11d3926de Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:26 +0100 Subject: [PATCH 056/144] python: build extension as itchcpp._itchcpp --- python/CMakeLists.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index c6d6d01..2e168e5 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -4,12 +4,16 @@ cmake_minimum_required(VERSION 3.25) # resolved through vcpkg or a system install. find_package(pybind11 CONFIG REQUIRED) +# The compiled extension is the private `_itchcpp` module imported by the pure +# Python `itchcpp` package, which re-exports it under the upstream-compatible +# `itchcpp.messages` / `itchcpp.parser` / `itchcpp.indicators` layout. pybind11_add_module(itchcpp_python src/bindings.cpp) -set_target_properties(itchcpp_python PROPERTIES OUTPUT_NAME "itchcpp") +set_target_properties(itchcpp_python PROPERTIES OUTPUT_NAME "_itchcpp") target_link_libraries(itchcpp_python PRIVATE itch::itch) -# When built through scikit-build-core (pip install .) install into the wheel. +# When built through scikit-build-core (pip install .) install the extension into +# the itchcpp package directory alongside the pure-Python modules. if(DEFINED SKBUILD) - install(TARGETS itchcpp_python LIBRARY DESTINATION .) + install(TARGETS itchcpp_python LIBRARY DESTINATION itchcpp) endif() From e898013347400e8fd045b16866061752df43707a Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:29 +0100 Subject: [PATCH 057/144] docs: rewrite python README for itchcpp package layout --- python/README.md | 140 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 102 insertions(+), 38 deletions(-) diff --git a/python/README.md b/python/README.md index a128100..71a9a1b 100644 --- a/python/README.md +++ b/python/README.md @@ -1,55 +1,119 @@ -# itchcpp — Python bindings +# itchcpp — a fast, drop-in backend for the pure-Python `itch` package `itchcpp` is a native (C++) NASDAQ TotalView-ITCH 5.0 parser, order-book engine, -and analytics layer exposed to Python via pybind11. It is designed as a faster, -drop-in **backend for the pure-Python [`itch`](https://github.com/bbalouki/itch) -package**: it mirrors that package's `MessageParser` API and message-class field -and `decode_price` semantics, so existing code can switch to the native parser -with minimal changes. +and analytics layer exposed to Python via pybind11. It is built to be a +**drop-in, faster backend** for the pure-Python +[`itch`](https://github.com/bbalouki/itch) package (published on PyPI as +[`itchfeed`](https://pypi.org/project/itchfeed/)). -## Build / install +The pure-Python parser unpacks every message with `struct.unpack` and materializes a +Python object in interpreter code, which is slow on full-day feeds. `itchcpp` does the +framing and field decoding in C++ (gigabytes per second) while exposing the **same +public API**: the same message classes (same names, same raw `bytes`/`int` +attributes), the same `MessageParser` semantics, the same `MarketMessage` helpers, +the same constants, and the same `create_message` factory. -The bindings require a C++20 compiler and pybind11. From the repository root: +## Drop-in usage + +The package layout mirrors `itch.*`, so migrating only changes the import root: + +```python +# pure-Python # native backend +from itch.parser import MessageParser from itchcpp.parser import MessageParser +from itch.messages import create_message from itchcpp.messages import create_message +from itch.indicators import MARKET_CATEGORY from itchcpp.indicators import MARKET_CATEGORY +``` + +```python +from itchcpp.parser import MessageParser +from itchcpp.messages import AddOrderNoMPIAttributionMessage + +parser = MessageParser() # or MessageParser(message_type=b"AFE") to filter types +with open("01302020.NASDAQ_ITCH50", "rb") as itch_file: + for message in parser.parse_file(itch_file): + if isinstance(message, AddOrderNoMPIAttributionMessage): + # Raw attributes are bytes/int, exactly like the pure-Python package. + print(message.stock, message.decode_price("price"), message.shares) + # decode() returns a human-readable dataclass (symbols trimmed, + # prices scaled): + print(message.decode()) +``` + +`parse_file` takes an open **binary file object** (so `gzip.open(path, "rb")` works +too); `parse_stream(raw_bytes)` parses an in-memory buffer. Both return lazy +iterators, apply the `message_type` filter, and stop at the end-of-messages system +event (`S` with `event_code == b"C"`). + +## What is exposed + +- **`itchcpp.messages`** — every message class under its exact upstream name + (`SystemEventMessage`, `StockDirectoryMessage`, `AddOrderNoMPIAttributionMessage`, + `AddOrderMPIDAttribution`, `OrderExecutedMessage`, `NonCrossTradeMessage`, + `CrossTradeMessage`, `NOIIMessage`, `MWCBDeclineLeveMessage`, + `RetailPriceImprovementIndicator`, `DLCRMessage`, ...); the abstract bases + `MarketMessage`, `AddOrderMessage`, `ModifyOrderMessage`, `TradeMessage` (the + concrete classes are registered as virtual subclasses, so `isinstance` works); the + `messages` registry (`dict[bytes, type]`); the `AllMessages` constant; and the + `create_message(message_type, **kwargs)` factory. +- **`MarketMessage` helpers** on every message: `decode()`, `decode_price(name)`, + `to_bytes()`, `set_timestamp(ts1, ts2)`, `split_timestamp()`, + `get_attributes(call_able=False)`, plus the `message_type`, `description`, + `message_size`, and `price_precision` metadata. +- **`itchcpp.parser.MessageParser`** — `parse_file`, `parse_stream`, + `parse_messages(data, callback)`, `get_message_type(bytes)`, and the `message_type` + filter. +- **`itchcpp.indicators`** — the field code lookup tables (`SYSTEM_EVENT_CODES`, + `MARKET_CATEGORY`, `TRADING_STATES`, `ISSUE_SUB_TYPE_VALUES`, ...). +- **Native extras** (not part of the pure-Python API): `itchcpp.book` + (`BookManager`, `L3Book`, `Bbo`) for single-pass order-book reconstruction, and + `itchcpp.analytics` (`Vwap`). + +## Install + +Prebuilt wheels are published to PyPI for Linux, macOS, and Windows (CPython +3.9-3.13), so most users need no compiler: ```bash -pip install ./python +pip install itchcpp ``` -This invokes scikit-build-core, which configures the parent CMake project with -`-DITCH_BUILD_PYTHON=ON` and builds the `itchcpp` extension module. +### Build from source -To build just the extension with CMake directly: +Building from source needs a C++20 compiler and pybind11 (pulled in automatically as +a build dependency). The `pyproject.toml` lives at the repository root, so build from +there: ```bash -cmake -S . -B build -DITCH_BUILD_PYTHON=ON \ - -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake -cmake --build build --target itchcpp_python +pip install . ``` -## Usage +This invokes scikit-build-core, which configures the CMake project with +`-DITCH_BUILD_PYTHON=ON` and builds the `itchcpp._itchcpp` extension module into the +`itchcpp` package. To build just the extension with CMake directly: -```python -import itchcpp - -parser = itchcpp.MessageParser() -for message in parser.parse_file("01302020.NASDAQ_ITCH50"): - if isinstance(message, itchcpp.AddOrderMessage): - print(message.stock, message.decode_price("price"), message.shares) - -# Full-market book reconstruction in one pass: -manager = itchcpp.BookManager() -with open("01302020.NASDAQ_ITCH50", "rb") as handle: - manager.process_stream(handle.read()) -book = manager.book_for_symbol("AAPL") -print(book.bbo().bid_price, book.bbo().ask_price) +```bash +cmake -S . -B build -DITCH_BUILD_PYTHON=ON +cmake --build build --target itchcpp_python ``` -## Relationship to the `itch` package +## Relationship to the `itch` (itchfeed) package + +`itchcpp` ships as a separate package and does not replace `itchfeed`; it exposes the +same API under the `itchcpp` import root. There are two migration paths: + +1. **Switch the import root** in your own code (`itch` -> `itchcpp`). +2. **Wire it as an optional backend** in `itchfeed`, so installed users get a + transparent speedup with no code change: + + ```python + try: + from itchcpp.parser import MessageParser # fast C++ backend + from itchcpp.messages import create_message, messages + except ImportError: # fall back to the pure-Python implementation + from itch.parser import MessageParser + from itch.messages import create_message, messages + ``` -The pure-Python `itch` package is already published on PyPI. Rather than rename or -break it, `itchcpp` ships as a separate, native package exposing the same API -surface (`MessageParser.parse_file`/`parse_stream`, typed message classes, -`decode_price`). The recommended migration is to have `itch` optionally import -`itchcpp` as its parsing backend when installed, falling back to the pure-Python -implementation otherwise — giving existing users a transparent speedup without a -breaking change. +The native message objects expose **raw** field values (`bytes` for character fields, +`int` for prices and quantities) just like the pure-Python classes, so existing code +that calls `decode_price(...)` or `decode()` keeps working unchanged. From 8d6baee8b2f0098d79d98f376936a39a707b38f8 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:33 +0100 Subject: [PATCH 058/144] python: mirror itchfeed message/parser API in native bindings --- python/src/bindings.cpp | 923 ++++++++++++++++++++++++++++++---------- 1 file changed, 693 insertions(+), 230 deletions(-) diff --git a/python/src/bindings.cpp b/python/src/bindings.cpp index 58535be..51d1513 100644 --- a/python/src/bindings.cpp +++ b/python/src/bindings.cpp @@ -1,8 +1,25 @@ +// Native (C++) backend for the pure-Python `itch` (PyPI: `itchfeed`) package. +// +// This module, compiled as `itchcpp._itchcpp`, mirrors the upstream public API +// exactly so it can serve as a faster drop-in: the same message classes (same +// names, same raw `bytes`/`int` attributes), the same `MessageParser` semantics +// (a `message_type` filter, lazy iteration, stop on the end-of-messages system +// event), the same `MarketMessage` helpers (`decode`, `decode_price`, +// `get_attributes`, `set_timestamp`, `split_timestamp`, `to_bytes`), and the same +// module members (`AllMessages`, `messages`, `create_message`). The pure-Python +// `itchcpp` package layered on top re-exports these under `itchcpp.messages`, +// `itchcpp.parser`, and `itchcpp.indicators`. + #include #include -#include +#include +#include +#include +#include +#include #include +#include #include #include @@ -16,344 +33,790 @@ namespace py = pybind11; namespace { -// A thin parser facade mirroring the pure-Python `itch` package's MessageParser, -// so this native module can serve as a drop-in, faster backend for it. -struct MessageParser {}; +// The set of every known message-type byte, in the upstream order. Mirrors +// `itch.messages.AllMessages` and doubles as the default parser filter. +constexpr std::string_view ALL_MESSAGES = "SAFECXDUBHRYPQINLVWKJhO"; + +// The width of the 2-byte big-endian length prefix that frames each message. +constexpr std::size_t FRAME_HEADER_LEN = 2; + +// Default chunk size for reading a file-backed stream (matches upstream's 64 KiB). +constexpr std::size_t DEFAULT_CACHE_SIZE = 65536; + +// Generic message helpers +// These operate on the Python object (`self`) so a single implementation serves +// every message class, exactly replicating the upstream `MarketMessage` methods. + +// Returns whether a looked-up attribute value is callable (a bound method). +auto is_callable(const py::object& value) -> bool { return PyCallable_Check(value.ptr()) != 0; } + +// decode_price(name): divide the named raw integer field by 10**price_precision, +// matching `MarketMessage.decode_price`. Raises ValueError for a method name and +// (via Python) TypeError when the attribute is not numeric. +auto message_decode_price(const py::object& self, const std::string& price_attr) -> py::object { + const py::object value = self.attr(price_attr.c_str()); + if (is_callable(value)) { + throw py::value_error("Please check the price attribute for " + price_attr); + } + const py::object precision = self.attr("price_precision"); + const py::object divisor = py::int_(10).attr("__pow__")(precision); + return value / divisor; +} + +// set_timestamp(ts1, ts2): reconstruct the 48-bit timestamp from two 32-bit halves. +auto message_set_timestamp(const py::object& self, std::uint64_t ts1, std::uint64_t ts2) -> void { + self.attr("timestamp") = py::int_(ts2 | (ts1 << 32U)); +} + +// split_timestamp(): split the timestamp into (high 32 bits, low 32 bits). +auto message_split_timestamp(const py::object& self) -> py::tuple { + const auto timestamp = self.attr("timestamp").cast(); + const std::uint64_t high {timestamp >> 32U}; + const std::uint64_t low {timestamp - (high << 32U)}; + return py::make_tuple(high, low); +} -// Standard price divisor (4 implied decimals); MWCB levels override this. -constexpr double STANDARD_DIVISOR = 10000.0; +// get_attributes(call_able): the non-callable (data) or callable (method) public +// attribute names, derived from dir(self) just like the upstream method. +auto message_get_attributes(const py::object& self, bool call_able) -> py::list { + py::list out; + const py::object names = py::module_::import("builtins").attr("dir")(self); + for (const auto& name : names) { + const auto text = name.cast(); + if (text.rfind("__", 0) == 0) { + continue; + } + if (is_callable(self.attr(name)) == call_able) { + out.append(name); + } + } + return out; +} -// Binds the fields common to every ITCH message plus the shared helpers. +// decode(prefix): build a human-readable dataclass named `{prefix}{ClassName}`, +// turning bytes fields into rstripped ASCII strings and price fields into floats. +auto message_decode(const py::object& self, const std::string& prefix) -> py::object { + const py::object builtins = py::module_::import("builtins"); + py::dict attrs; + for (const auto& name : builtins.attr("dir")(self)) { + const auto text = name.cast(); + if (text.rfind("__", 0) == 0) { + continue; + } + py::object value = self.attr(name); + if (text.find("price") != std::string::npos && text != "price_precision" && + text != "decode_price") { + value = self.attr("decode_price")(name); + } + if (is_callable(value)) { + continue; + } + if (py::isinstance(value)) { + attrs[name] = value.attr("decode")("ascii").attr("rstrip")(); + } else if ( + py::isinstance(value) || py::isinstance(value) || + py::isinstance(value) || py::isinstance(value) + ) { + attrs[name] = value; + } + } + + py::list fields; + for (const auto& item : attrs) { + fields.append(py::make_tuple(item.first, builtins.attr("type")(item.second))); + } + const auto class_name = prefix + self.attr("__class__").attr("__name__").cast(); + const py::object make_dataclass = py::module_::import("dataclasses").attr("make_dataclass"); + return make_dataclass(class_name, fields)(**attrs); +} + +// Field exposure helpers + +// Exposes a single-character field as a length-1 `bytes` value (read/write). template -auto bind_common(py::class_& cls, double price_divisor = STANDARD_DIVISOR) -> void { - cls.def_property_readonly("message_type", [](const MsgType& msg) { - return std::string(1, msg.message_type); - }); - cls.def_readonly("stock_locate", &MsgType::stock_locate); - cls.def_readonly("tracking_number", &MsgType::tracking_number); - cls.def_readonly("timestamp", &MsgType::timestamp); - // decode_price(name) divides the named raw integer field by the scale, exactly - // like the upstream package's MarketMessage.decode_price. - cls.def( - "decode_price", - [price_divisor](const py::object& self, const std::string& name) { - return self.attr(name.c_str()).cast() / price_divisor; - }, - py::arg("attribute_name") +auto def_byte(py::class_& cls, const char* name, char MsgType::* member) -> void { + cls.def_property( + name, + [member](const MsgType& msg) { return py::bytes(&(msg.*member), 1); }, + [member](MsgType& msg, const py::bytes& value) { + const std::string text {value}; + msg.*member = text.empty() ? '\0' : text.front(); + } + ); +} + +// Exposes a fixed-width character array as a `bytes` value of that width (the raw, +// space-padded symbol, not trimmed), matching upstream attribute semantics. +template +auto def_byte_array(py::class_& cls, const char* name, char (MsgType::*member)[Width]) + -> void { + cls.def_property( + name, + [member](const MsgType& msg) { return py::bytes(msg.*member, Width); }, + [member](MsgType& msg, const py::bytes& value) { + const std::string text {value}; + std::fill(std::begin(msg.*member), std::end(msg.*member), '\0'); + std::memcpy(msg.*member, text.data(), std::min(text.size(), Width)); + } ); - // to_bytes() serializes the message back to wire form, matching the upstream - // MarketMessage.to_bytes (here it returns the message body without the frame - // length prefix). - cls.def("to_bytes", [](const MsgType& msg) { - const std::vector bytes = itch::encode_message(itch::Message {msg}); - return py::bytes(reinterpret_cast(bytes.data()), bytes.size()); // NOLINT - }); } -// Exposes an 8-char stock symbol field as a trimmed Python string. +// Serializes a message back to its wire body (type byte first, no frame length). template -auto stock_property(py::class_& cls) -> void { - cls.def_property_readonly("stock", [](const MsgType& msg) { - return itch::to_string(msg.stock, itch::STOCK_LEN); +auto message_to_bytes(const MsgType& msg) -> py::bytes { + const std::vector bytes = itch::encode_message(itch::Message {msg}); + return {reinterpret_cast(bytes.data()), bytes.size()}; // NOLINT +} + +// Frames a body so the core parser (which expects a 2-byte big-endian length) can +// decode it: for every real ITCH message (< 256 bytes) this is byte-identical to +// the upstream `\x00` + length-byte header. +auto unpack_body(const std::string& body) -> itch::Message { + std::string frame; + frame.reserve(FRAME_HEADER_LEN + body.size()); + frame.push_back(static_cast((body.size() >> 8U) & 0xFFU)); + frame.push_back(static_cast(body.size() & 0xFFU)); + frame.append(body); + + itch::Parser parser; + std::optional result; + parser.parse(frame.data(), frame.size(), [&result](const itch::Message& message) { + result = message; }); + if (!result.has_value()) { + throw py::value_error("Could not unpack message body"); + } + return *result; +} + +// Casts a parsed variant alternative to its concrete Python message object. +auto message_to_object(const itch::Message& message) -> py::object { + return std::visit([](const auto& concrete) { return py::cast(concrete); }, message); } -auto parse_buffer(const std::string& data) -> py::list { - py::list out; - itch::Parser parser; - parser.parse(data.data(), data.size(), [&out](const itch::Message& message) { - std::visit([&out](const auto& concrete) { out.append(py::cast(concrete)); }, message); +// Binds the fields and helpers common to every message class. `precision` is 4 for +// all messages except the MWCB decline levels (8). Returns the class so the caller +// can append its message-specific fields. +template +auto bind_common(py::module_& mod, const char* name, const char* description, int precision) + -> py::class_ { + py::class_ cls {mod, name}; + cls.def(py::init<>()); + cls.def(py::init([](const py::bytes& body) { + return std::get(unpack_body(std::string {body})); + })); + + cls.def_property_readonly("message_type", [](const MsgType& msg) { + return py::bytes(&msg.message_type, 1); }); - return out; + cls.def_readwrite("stock_locate", &MsgType::stock_locate); + cls.def_readwrite("tracking_number", &MsgType::tracking_number); + cls.def_readwrite("timestamp", &MsgType::timestamp); + + cls.def("decode_price", &message_decode_price, py::arg("price_attr")); + cls.def("set_timestamp", &message_set_timestamp, py::arg("ts1"), py::arg("ts2")); + cls.def("split_timestamp", &message_split_timestamp); + cls.def("get_attributes", &message_get_attributes, py::arg("call_able") = false); + cls.def("decode", &message_decode, py::arg("prefix") = std::string {}); + cls.def("to_bytes", &message_to_bytes); + cls.def("__bytes__", &message_to_bytes); + cls.def("__repr__", [](const py::object& self) { return py::repr(self.attr("decode")()); }); + + cls.attr("description") = py::str(description); + cls.attr("price_precision") = py::int_(precision); + cls.attr("message_size") = py::int_(itch::encode_message(itch::Message {MsgType {}}).size()); + return cls; } +constexpr int STANDARD_PRECISION = 4; +constexpr int MWCB_PRECISION = 8; + +// A factory that default-constructs each message type for `create_message`. +using Factory = std::function; + } // namespace -PYBIND11_MODULE(itchcpp, mod) { +PYBIND11_MODULE(_itchcpp, mod) { mod.doc() = - "Native (C++) NASDAQ TotalView-ITCH 5.0 parser. A faster drop-in backend " - "for the pure-Python `itch` package: it exposes a MessageParser plus typed " - "message classes with the same field and decode_price semantics."; + "Native (C++) NASDAQ TotalView-ITCH 5.0 backend for the pure-Python `itch` " + "(itchfeed) package: a MessageParser plus typed message classes with matching " + "raw attributes and decode/decode_price/to_bytes semantics."; + + // Maps a message-type byte to its Python class and to a default-constructing + // factory, powering the module-level `messages` dict and `create_message`. + auto type_to_class = std::make_shared(); + auto type_to_factory = std::make_shared>(); - // --- Message classes ----------------------------------------------------- + const auto register_message = [&](char type_byte, const py::object& cls, Factory factory) { + (*type_to_class)[py::bytes(&type_byte, 1)] = cls; + (*type_to_factory)[type_byte] = std::move(factory); + }; + + // Message classes { - py::class_ cls {mod, "SystemEventMessage"}; - bind_common(cls); - cls.def_property_readonly("event_code", [](const itch::SystemEventMessage& m) { - return std::string(1, m.event_code); - }); + auto cls = bind_common( + mod, "SystemEventMessage", "System Event Message", STANDARD_PRECISION + ); + def_byte(cls, "event_code", &itch::SystemEventMessage::event_code); + register_message('S', cls, [] { return py::cast(itch::SystemEventMessage {}); }); } { - py::class_ cls {mod, "StockDirectoryMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_readonly("round_lot_size", &itch::StockDirectoryMessage::round_lot_size); - cls.def_property_readonly("market_category", [](const itch::StockDirectoryMessage& m) { - return std::string(1, m.market_category); - }); + auto cls = bind_common( + mod, "StockDirectoryMessage", "Stock Directory Message", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::StockDirectoryMessage::stock); + def_byte(cls, "market_category", &itch::StockDirectoryMessage::market_category); + def_byte( + cls, + "financial_status_indicator", + &itch::StockDirectoryMessage::financial_status_indicator + ); + cls.def_readwrite("round_lot_size", &itch::StockDirectoryMessage::round_lot_size); + def_byte(cls, "round_lots_only", &itch::StockDirectoryMessage::round_lots_only); + def_byte(cls, "issue_classification", &itch::StockDirectoryMessage::issue_classification); + def_byte_array(cls, "issue_sub_type", &itch::StockDirectoryMessage::issue_sub_type); + def_byte(cls, "authenticity", &itch::StockDirectoryMessage::authenticity); + def_byte( + cls, + "short_sale_threshold_indicator", + &itch::StockDirectoryMessage::short_sale_threshold_indicator + ); + def_byte(cls, "ipo_flag", &itch::StockDirectoryMessage::ipo_flag); + def_byte(cls, "luld_ref", &itch::StockDirectoryMessage::luld_ref); + def_byte(cls, "etp_flag", &itch::StockDirectoryMessage::etp_flag); + cls.def_readwrite("etp_leverage_factor", &itch::StockDirectoryMessage::etp_leverage_factor); + def_byte(cls, "inverse_indicator", &itch::StockDirectoryMessage::inverse_indicator); + register_message('R', cls, [] { return py::cast(itch::StockDirectoryMessage {}); }); } { - py::class_ cls {mod, "StockTradingActionMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_property_readonly("trading_state", [](const itch::StockTradingActionMessage& m) { - return std::string(1, m.trading_state); - }); - cls.def_property_readonly("reason", [](const itch::StockTradingActionMessage& m) { - return itch::to_string(m.reason, 4); - }); + auto cls = bind_common( + mod, "StockTradingActionMessage", "Stock Trading Action Message", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::StockTradingActionMessage::stock); + def_byte(cls, "trading_state", &itch::StockTradingActionMessage::trading_state); + def_byte(cls, "reserved", &itch::StockTradingActionMessage::reserved); + def_byte_array(cls, "reason", &itch::StockTradingActionMessage::reason); + register_message('H', cls, [] { return py::cast(itch::StockTradingActionMessage {}); }); } { - py::class_ cls {mod, "RegSHOMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_property_readonly("reg_sho_action", [](const itch::RegSHOMessage& m) { - return std::string(1, m.reg_sho_action); - }); + auto cls = bind_common( + mod, + "RegSHOMessage", + "Reg SHO Short Sale Price Test Restricted Indicator", + STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::RegSHOMessage::stock); + def_byte(cls, "reg_sho_action", &itch::RegSHOMessage::reg_sho_action); + register_message('Y', cls, [] { return py::cast(itch::RegSHOMessage {}); }); } { - py::class_ cls { - mod, "MarketParticipantPositionMessage" - }; - bind_common(cls); - stock_property(cls); - cls.def_property_readonly("mpid", [](const itch::MarketParticipantPositionMessage& m) { - return itch::to_string(m.mpid, 4); + auto cls = bind_common( + mod, + "MarketParticipantPositionMessage", + "Market Participant Position message", + STANDARD_PRECISION + ); + def_byte_array(cls, "mpid", &itch::MarketParticipantPositionMessage::mpid); + def_byte_array(cls, "stock", &itch::MarketParticipantPositionMessage::stock); + def_byte( + cls, + "primary_market_maker", + &itch::MarketParticipantPositionMessage::primary_market_maker + ); + def_byte( + cls, "market_maker_mode", &itch::MarketParticipantPositionMessage::market_maker_mode + ); + def_byte( + cls, + "market_participant_state", + &itch::MarketParticipantPositionMessage::market_participant_state + ); + register_message('L', cls, [] { + return py::cast(itch::MarketParticipantPositionMessage {}); }); } { - py::class_ cls {mod, "MWCBDeclineLevelMessage"}; - bind_common(cls, 1.0E8); // MWCB levels carry 8 implied decimals. - cls.def_readonly("level1", &itch::MWCBDeclineLevelMessage::level1); - cls.def_readonly("level2", &itch::MWCBDeclineLevelMessage::level2); - cls.def_readonly("level3", &itch::MWCBDeclineLevelMessage::level3); + // Upstream spells this class `MWCBDeclineLeveMessage` (sic) and names the + // levels `levelN_price`; mirror both for drop-in compatibility. + auto cls = bind_common( + mod, + "MWCBDeclineLeveMessage", + "Market wide circuit breaker Decline Level Message", + MWCB_PRECISION + ); + cls.def_readwrite("level1_price", &itch::MWCBDeclineLevelMessage::level1); + cls.def_readwrite("level2_price", &itch::MWCBDeclineLevelMessage::level2); + cls.def_readwrite("level3_price", &itch::MWCBDeclineLevelMessage::level3); + register_message('V', cls, [] { return py::cast(itch::MWCBDeclineLevelMessage {}); }); } { - py::class_ cls {mod, "MWCBStatusMessage"}; - bind_common(cls); - cls.def_property_readonly("breached_level", [](const itch::MWCBStatusMessage& m) { - return std::string(1, m.breached_level); - }); + auto cls = bind_common( + mod, + "MWCBStatusMessage", + "Market-Wide Circuit Breaker Status message", + STANDARD_PRECISION + ); + def_byte(cls, "breached_level", &itch::MWCBStatusMessage::breached_level); + register_message('W', cls, [] { return py::cast(itch::MWCBStatusMessage {}); }); } { - py::class_ cls {mod, "IPOQuotingPeriodUpdateMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_readonly("ipo_price", &itch::IPOQuotingPeriodUpdateMessage::ipo_price); - cls.def_readonly( - "ipo_quotation_release_time", - &itch::IPOQuotingPeriodUpdateMessage::ipo_quotation_release_time + auto cls = bind_common( + mod, + "IPOQuotingPeriodUpdateMessage", + "IPO Quoting Period Update Message", + STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::IPOQuotingPeriodUpdateMessage::stock); + cls.def_readwrite( + "ipo_release_time", &itch::IPOQuotingPeriodUpdateMessage::ipo_quotation_release_time ); + def_byte( + cls, + "ipo_release_qualifier", + &itch::IPOQuotingPeriodUpdateMessage::ipo_quotation_release_qualifier + ); + cls.def_readwrite("ipo_price", &itch::IPOQuotingPeriodUpdateMessage::ipo_price); + register_message('K', cls, [] { return py::cast(itch::IPOQuotingPeriodUpdateMessage {}); }); } { - py::class_ cls {mod, "LULDAuctionCollarMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_readonly( + auto cls = bind_common( + mod, "LULDAuctionCollarMessage", "LULD Auction Collar", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::LULDAuctionCollarMessage::stock); + cls.def_readwrite( "auction_collar_reference_price", &itch::LULDAuctionCollarMessage::auction_collar_reference_price ); - cls.def_readonly( + cls.def_readwrite( "upper_auction_collar_price", &itch::LULDAuctionCollarMessage::upper_auction_collar_price ); - cls.def_readonly( + cls.def_readwrite( "lower_auction_collar_price", &itch::LULDAuctionCollarMessage::lower_auction_collar_price ); + // Upstream attribute keeps the original `extention` spelling. + cls.def_readwrite( + "auction_collar_extention", &itch::LULDAuctionCollarMessage::auction_collar_extension + ); + register_message('J', cls, [] { return py::cast(itch::LULDAuctionCollarMessage {}); }); } { - py::class_ cls {mod, "OperationalHaltMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_property_readonly("market_code", [](const itch::OperationalHaltMessage& m) { - return std::string(1, m.market_code); - }); - cls.def_property_readonly( - "operational_halt_action", [](const itch::OperationalHaltMessage& m) { - return std::string(1, m.operational_halt_action); - } + auto cls = bind_common( + mod, "OperationalHaltMessage", "Operational Halt", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::OperationalHaltMessage::stock); + def_byte(cls, "market_code", &itch::OperationalHaltMessage::market_code); + def_byte( + cls, "operational_halt_action", &itch::OperationalHaltMessage::operational_halt_action ); + register_message('h', cls, [] { return py::cast(itch::OperationalHaltMessage {}); }); } { - py::class_ cls {mod, "AddOrderMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_readonly("order_reference_number", &itch::AddOrderMessage::order_reference_number); - cls.def_readonly("shares", &itch::AddOrderMessage::shares); - cls.def_readonly("price", &itch::AddOrderMessage::price); - cls.def_property_readonly("buy_sell_indicator", [](const itch::AddOrderMessage& m) { - return std::string(1, m.buy_sell_indicator); - }); + auto cls = bind_common( + mod, + "AddOrderNoMPIAttributionMessage", + "Add Order - No MPID Attribution Message", + STANDARD_PRECISION + ); + cls.def_readwrite("order_reference_number", &itch::AddOrderMessage::order_reference_number); + def_byte(cls, "buy_sell_indicator", &itch::AddOrderMessage::buy_sell_indicator); + cls.def_readwrite("shares", &itch::AddOrderMessage::shares); + def_byte_array(cls, "stock", &itch::AddOrderMessage::stock); + cls.def_readwrite("price", &itch::AddOrderMessage::price); + register_message('A', cls, [] { return py::cast(itch::AddOrderMessage {}); }); } { - py::class_ cls { - mod, "AddOrderMPIDAttributionMessage" - }; - bind_common(cls); - stock_property(cls); - cls.def_readonly( + auto cls = bind_common( + mod, + "AddOrderMPIDAttribution", + "Add Order - MPID Attribution Message", + STANDARD_PRECISION + ); + cls.def_readwrite( "order_reference_number", &itch::AddOrderMPIDAttributionMessage::order_reference_number ); - cls.def_readonly("shares", &itch::AddOrderMPIDAttributionMessage::shares); - cls.def_readonly("price", &itch::AddOrderMPIDAttributionMessage::price); - cls.def_property_readonly( - "buy_sell_indicator", [](const itch::AddOrderMPIDAttributionMessage& m) { - return std::string(1, m.buy_sell_indicator); - } + def_byte( + cls, "buy_sell_indicator", &itch::AddOrderMPIDAttributionMessage::buy_sell_indicator ); - cls.def_property_readonly("attribution", [](const itch::AddOrderMPIDAttributionMessage& m) { - return itch::to_string(m.attribution, 4); + cls.def_readwrite("shares", &itch::AddOrderMPIDAttributionMessage::shares); + def_byte_array(cls, "stock", &itch::AddOrderMPIDAttributionMessage::stock); + cls.def_readwrite("price", &itch::AddOrderMPIDAttributionMessage::price); + def_byte_array(cls, "attribution", &itch::AddOrderMPIDAttributionMessage::attribution); + register_message('F', cls, [] { + return py::cast(itch::AddOrderMPIDAttributionMessage {}); }); } { - py::class_ cls {mod, "OrderExecutedMessage"}; - bind_common(cls); - cls.def_readonly( + auto cls = bind_common( + mod, "OrderExecutedMessage", "Add Order - Order Executed Message", STANDARD_PRECISION + ); + cls.def_readwrite( "order_reference_number", &itch::OrderExecutedMessage::order_reference_number ); - cls.def_readonly("executed_shares", &itch::OrderExecutedMessage::executed_shares); - cls.def_readonly("match_number", &itch::OrderExecutedMessage::match_number); + cls.def_readwrite("executed_shares", &itch::OrderExecutedMessage::executed_shares); + cls.def_readwrite("match_number", &itch::OrderExecutedMessage::match_number); + register_message('E', cls, [] { return py::cast(itch::OrderExecutedMessage {}); }); } { - py::class_ cls {mod, "OrderExecutedWithPriceMessage"}; - bind_common(cls); - cls.def_readonly( + auto cls = bind_common( + mod, + "OrderExecutedWithPriceMessage", + "Add Order - Order Executed with Price Message", + STANDARD_PRECISION + ); + cls.def_readwrite( "order_reference_number", &itch::OrderExecutedWithPriceMessage::order_reference_number ); - cls.def_readonly("executed_shares", &itch::OrderExecutedWithPriceMessage::executed_shares); - cls.def_readonly("match_number", &itch::OrderExecutedWithPriceMessage::match_number); - cls.def_readonly("execution_price", &itch::OrderExecutedWithPriceMessage::execution_price); - cls.def_property_readonly("printable", [](const itch::OrderExecutedWithPriceMessage& m) { - return std::string(1, m.printable); - }); + cls.def_readwrite("executed_shares", &itch::OrderExecutedWithPriceMessage::executed_shares); + cls.def_readwrite("match_number", &itch::OrderExecutedWithPriceMessage::match_number); + def_byte(cls, "printable", &itch::OrderExecutedWithPriceMessage::printable); + cls.def_readwrite("execution_price", &itch::OrderExecutedWithPriceMessage::execution_price); + register_message('C', cls, [] { return py::cast(itch::OrderExecutedWithPriceMessage {}); }); } { - py::class_ cls {mod, "OrderCancelMessage"}; - bind_common(cls); - cls.def_readonly( + auto cls = bind_common( + mod, "OrderCancelMessage", "Order Cancel Message", STANDARD_PRECISION + ); + cls.def_readwrite( "order_reference_number", &itch::OrderCancelMessage::order_reference_number ); - cls.def_readonly("cancelled_shares", &itch::OrderCancelMessage::cancelled_shares); + cls.def_readwrite("cancelled_shares", &itch::OrderCancelMessage::cancelled_shares); + register_message('X', cls, [] { return py::cast(itch::OrderCancelMessage {}); }); } { - py::class_ cls {mod, "OrderDeleteMessage"}; - bind_common(cls); - cls.def_readonly( + auto cls = bind_common( + mod, "OrderDeleteMessage", "Order Delete Message", STANDARD_PRECISION + ); + cls.def_readwrite( "order_reference_number", &itch::OrderDeleteMessage::order_reference_number ); + register_message('D', cls, [] { return py::cast(itch::OrderDeleteMessage {}); }); } { - py::class_ cls {mod, "OrderReplaceMessage"}; - bind_common(cls); - cls.def_readonly( - "original_order_reference_number", - &itch::OrderReplaceMessage::original_order_reference_number + auto cls = bind_common( + mod, "OrderReplaceMessage", "Order Replace Message", STANDARD_PRECISION ); - cls.def_readonly( + cls.def_readwrite( + "order_reference_number", &itch::OrderReplaceMessage::original_order_reference_number + ); + cls.def_readwrite( "new_order_reference_number", &itch::OrderReplaceMessage::new_order_reference_number ); - cls.def_readonly("shares", &itch::OrderReplaceMessage::shares); - cls.def_readonly("price", &itch::OrderReplaceMessage::price); + cls.def_readwrite("shares", &itch::OrderReplaceMessage::shares); + cls.def_readwrite("price", &itch::OrderReplaceMessage::price); + register_message('U', cls, [] { return py::cast(itch::OrderReplaceMessage {}); }); } { - py::class_ cls {mod, "NonCrossTradeMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_readonly( + auto cls = bind_common( + mod, "NonCrossTradeMessage", "Trade Message", STANDARD_PRECISION + ); + cls.def_readwrite( "order_reference_number", &itch::NonCrossTradeMessage::order_reference_number ); - cls.def_readonly("shares", &itch::NonCrossTradeMessage::shares); - cls.def_readonly("price", &itch::NonCrossTradeMessage::price); - cls.def_readonly("match_number", &itch::NonCrossTradeMessage::match_number); - cls.def_property_readonly("buy_sell_indicator", [](const itch::NonCrossTradeMessage& m) { - return std::string(1, m.buy_sell_indicator); - }); + def_byte(cls, "buy_sell_indicator", &itch::NonCrossTradeMessage::buy_sell_indicator); + cls.def_readwrite("shares", &itch::NonCrossTradeMessage::shares); + def_byte_array(cls, "stock", &itch::NonCrossTradeMessage::stock); + cls.def_readwrite("price", &itch::NonCrossTradeMessage::price); + cls.def_readwrite("match_number", &itch::NonCrossTradeMessage::match_number); + register_message('P', cls, [] { return py::cast(itch::NonCrossTradeMessage {}); }); } { - py::class_ cls {mod, "CrossTradeMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_readonly("shares", &itch::CrossTradeMessage::shares); - cls.def_readonly("cross_price", &itch::CrossTradeMessage::cross_price); - cls.def_readonly("match_number", &itch::CrossTradeMessage::match_number); - cls.def_property_readonly("cross_type", [](const itch::CrossTradeMessage& m) { - return std::string(1, m.cross_type); - }); + auto cls = bind_common( + mod, "CrossTradeMessage", "Cross Trade Message", STANDARD_PRECISION + ); + cls.def_readwrite("shares", &itch::CrossTradeMessage::shares); + def_byte_array(cls, "stock", &itch::CrossTradeMessage::stock); + cls.def_readwrite("cross_price", &itch::CrossTradeMessage::cross_price); + cls.def_readwrite("match_number", &itch::CrossTradeMessage::match_number); + def_byte(cls, "cross_type", &itch::CrossTradeMessage::cross_type); + register_message('Q', cls, [] { return py::cast(itch::CrossTradeMessage {}); }); } { - py::class_ cls {mod, "BrokenTradeMessage"}; - bind_common(cls); - cls.def_readonly("match_number", &itch::BrokenTradeMessage::match_number); + auto cls = bind_common( + mod, "BrokenTradeMessage", "Broken Trade Message", STANDARD_PRECISION + ); + cls.def_readwrite("match_number", &itch::BrokenTradeMessage::match_number); + register_message('B', cls, [] { return py::cast(itch::BrokenTradeMessage {}); }); } { - py::class_ cls {mod, "NOIIMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_readonly("paired_shares", &itch::NOIIMessage::paired_shares); - cls.def_readonly("imbalance_shares", &itch::NOIIMessage::imbalance_shares); - cls.def_readonly("far_price", &itch::NOIIMessage::far_price); - cls.def_readonly("near_price", &itch::NOIIMessage::near_price); - cls.def_readonly("current_reference_price", &itch::NOIIMessage::current_reference_price); - cls.def_property_readonly("imbalance_direction", [](const itch::NOIIMessage& m) { - return std::string(1, m.imbalance_direction); - }); - cls.def_property_readonly("cross_type", [](const itch::NOIIMessage& m) { - return std::string(1, m.cross_type); - }); + auto cls = + bind_common(mod, "NOIIMessage", "NOII Message", STANDARD_PRECISION); + cls.def_readwrite("paired_shares", &itch::NOIIMessage::paired_shares); + cls.def_readwrite("imbalance_shares", &itch::NOIIMessage::imbalance_shares); + def_byte(cls, "imbalance_direction", &itch::NOIIMessage::imbalance_direction); + def_byte_array(cls, "stock", &itch::NOIIMessage::stock); + cls.def_readwrite("far_price", &itch::NOIIMessage::far_price); + cls.def_readwrite("near_price", &itch::NOIIMessage::near_price); + cls.def_readwrite("current_reference_price", &itch::NOIIMessage::current_reference_price); + def_byte(cls, "cross_type", &itch::NOIIMessage::cross_type); + def_byte(cls, "variation_indicator", &itch::NOIIMessage::price_variation_indicator); + register_message('I', cls, [] { return py::cast(itch::NOIIMessage {}); }); } { - py::class_ cls { - mod, "RetailPriceImprovementIndicatorMessage" - }; - bind_common(cls); - stock_property(cls); - cls.def_property_readonly( - "interest_flag", [](const itch::RetailPriceImprovementIndicatorMessage& m) { - return std::string(1, m.interest_flag); - } + auto cls = bind_common( + mod, "RetailPriceImprovementIndicator", "Retail Interest message", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::RetailPriceImprovementIndicatorMessage::stock); + def_byte( + cls, "interest_flag", &itch::RetailPriceImprovementIndicatorMessage::interest_flag ); + register_message('N', cls, [] { + return py::cast(itch::RetailPriceImprovementIndicatorMessage {}); + }); } { - py::class_ cls {mod, "DLCRMessage"}; - bind_common(cls); - stock_property(cls); - cls.def_readonly("near_execution_price", &itch::DLCRMessage::near_execution_price); - cls.def_readonly("minimum_allowable_price", &itch::DLCRMessage::minimum_allowable_price); - cls.def_readonly("maximum_allowable_price", &itch::DLCRMessage::maximum_allowable_price); + auto cls = bind_common( + mod, "DLCRMessage", "Direct Listing with Capital Raise Message", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::DLCRMessage::stock); + def_byte(cls, "open_eligibility_status", &itch::DLCRMessage::open_eligibility_status); + cls.def_readwrite("minimum_allowable_price", &itch::DLCRMessage::minimum_allowable_price); + cls.def_readwrite("maximum_allowable_price", &itch::DLCRMessage::maximum_allowable_price); + cls.def_readwrite("near_execution_price", &itch::DLCRMessage::near_execution_price); + cls.def_readwrite("near_execution_time", &itch::DLCRMessage::near_execution_time); + cls.def_readwrite("lower_price_range_collar", &itch::DLCRMessage::lower_price_range_collar); + cls.def_readwrite("upper_price_range_collar", &itch::DLCRMessage::upper_price_range_collar); + register_message('O', cls, [] { return py::cast(itch::DLCRMessage {}); }); } - // --- Parser facade ------------------------------------------------------- + // Module-level message registry and factory + mod.attr("AllMessages") = py::bytes(std::string {ALL_MESSAGES}); + mod.attr("messages") = *type_to_class; + + mod.def( + "create_message", + [type_to_factory](const py::bytes& message_type, const py::kwargs& kwargs) -> py::object { + const std::string type_text {message_type}; + if (type_text.size() != 1) { + throw py::value_error("message_type must be a single byte"); + } + const auto factory = type_to_factory->find(type_text.front()); + if (factory == type_to_factory->end()) { + throw py::value_error("Unknown message type: " + type_text); + } + py::object instance = factory->second(); + constexpr std::uint64_t timestamp_mask {(std::uint64_t {1} << 48U) - 1U}; + for (const auto& item : kwargs) { + if (item.first.cast() == "timestamp") { + instance.attr("timestamp") = + py::int_(item.second.cast() & timestamp_mask); + } else { + instance.attr(item.first) = item.second; + } + } + return instance; + }, + py::arg("message_type"), + "Create a message of the given type byte, populated from keyword attributes." + ); + + // Parser + // A lazy iterator over framed ITCH messages, applying the type filter, the + // optional save_file re-emission, and the stop-on-system-event-'C' rule. When + // `m_file` is a binary file object the buffer is refilled `m_cachesize` bytes at + // a time (consumed bytes are dropped), so a whole-day feed never has to be held + // in memory at once; for an in-memory stream `m_file` is None. + struct MessageIterator { + std::string m_buffer; + std::size_t m_offset {0}; + std::string m_filter; + py::object m_file; + std::size_t m_cachesize {DEFAULT_CACHE_SIZE}; + py::object m_save_file; + bool m_stopped {false}; + }; + + py::class_(mod, "_MessageIterator") + .def("__iter__", [](py::object self) { return self; }) + .def("__next__", [](MessageIterator& iter) -> py::object { + // Ensures at least `need` unconsumed bytes are buffered, pulling more + // from the file (and dropping already-consumed bytes) when possible. + const auto fill_to = [&iter](std::size_t need) -> bool { + while (iter.m_buffer.size() - iter.m_offset < need) { + if (iter.m_file.is_none()) { + return false; + } + if (iter.m_offset > 0) { + iter.m_buffer.erase(0, iter.m_offset); + iter.m_offset = 0; + } + const std::string chunk { + iter.m_file.attr("read")(iter.m_cachesize).cast() + }; + if (chunk.empty()) { + return false; + } + iter.m_buffer += chunk; + } + return true; + }; + + while (true) { + if (iter.m_stopped || !fill_to(FRAME_HEADER_LEN)) { + throw py::stop_iteration(); + } + const auto high = static_cast(iter.m_buffer[iter.m_offset]); + const auto low = static_cast(iter.m_buffer[iter.m_offset + 1]); + const std::size_t body_len {(static_cast(high) << 8U) | low}; + const std::size_t total {FRAME_HEADER_LEN + body_len}; + // fill_to may compact the buffer (resetting m_offset), so read the + // frame position only after it returns. + if (!fill_to(total)) { + throw py::stop_iteration(); // Incomplete trailing message. + } + const std::size_t pos = iter.m_offset; + + const char type_byte = iter.m_buffer[pos + FRAME_HEADER_LEN]; + if (ALL_MESSAGES.find(type_byte) == std::string_view::npos) { + throw py::value_error(std::string {"Unknown message type: "} + type_byte); + } + + itch::Parser parser; + std::optional parsed; + parser.parse( + iter.m_buffer.data() + pos, total, [&parsed](const itch::Message& message) { + parsed = message; + } + ); + const std::string_view frame {iter.m_buffer.data() + pos, total}; + iter.m_offset = pos + total; + + const bool kept = iter.m_filter.find(type_byte) != std::string::npos; + const bool is_stop = type_byte == 'S' && + std::get(*parsed).event_code == 'C'; + + if (kept) { + if (!iter.m_save_file.is_none()) { + iter.m_save_file.attr("write")(py::bytes(frame.data(), frame.size())); + } + if (is_stop) { + iter.m_stopped = true; + } + return message_to_object(*parsed); + } + if (is_stop) { + throw py::stop_iteration(); + } + } + }); + + struct MessageParser { + std::string m_filter; + }; + + const auto make_iterator = [](const std::string& filter, + std::string buffer, + py::object file, + std::size_t cachesize, + const py::object& save_file) { + MessageIterator iter; + iter.m_buffer = std::move(buffer); + iter.m_filter = filter; + iter.m_file = std::move(file); + iter.m_cachesize = cachesize; + iter.m_save_file = save_file; + return iter; + }; + py::class_(mod, "MessageParser") - .def(py::init<>()) + .def( + py::init([](const py::bytes& message_type) { + return MessageParser {std::string {message_type}}; + }), + py::arg("message_type") = py::bytes(std::string {ALL_MESSAGES}) + ) + .def_property_readonly( + "message_type", [](const MessageParser& self) { return py::bytes(self.m_filter); } + ) + .def( + "get_message_type", + [](const MessageParser&, const py::bytes& message) { + const std::string body {message}; + if (body.empty() || ALL_MESSAGES.find(body.front()) == std::string_view::npos) { + throw py::value_error( + std::string {"Unknown message type: "} + (body.empty() ? '?' : body.front()) + ); + } + return message_to_object(unpack_body(body)); + }, + py::arg("message") + ) .def( "parse_stream", - [](MessageParser&, const py::bytes& data) { return parse_buffer(std::string {data}); }, - py::arg("raw_binary_data"), - "Parse messages from raw bytes, returning a list of message objects." + [make_iterator]( + const MessageParser& self, const py::bytes& data, const py::object& save_file + ) { + return make_iterator( + self.m_filter, std::string {data}, py::none(), DEFAULT_CACHE_SIZE, save_file + ); + }, + py::arg("data"), + py::arg("save_file") = py::none() ) .def( "parse_file", - [](MessageParser&, const std::string& path) { - std::ifstream file {path, std::ios::binary}; - const std::string data {std::istreambuf_iterator {file}, {}}; - return parse_buffer(data); + [make_iterator]( + const MessageParser& self, + const py::object& file, + std::size_t cachesize, + const py::object& save_file + ) { + // The file is read lazily, `cachesize` bytes at a time, by the iterator. + return make_iterator(self.m_filter, std::string {}, file, cachesize, save_file); + }, + py::arg("file"), + py::arg("cachesize") = DEFAULT_CACHE_SIZE, + py::arg("save_file") = py::none() + ) + .def( + "parse_messages", + [make_iterator]( + const MessageParser& self, const py::object& data, const py::object& callback + ) { + const bool is_buffer = + py::isinstance(data) || py::isinstance(data); + MessageIterator iter = + is_buffer + ? make_iterator( + self.m_filter, + std::string {py::bytes(data)}, + py::none(), + DEFAULT_CACHE_SIZE, + py::none() + ) + : make_iterator( + self.m_filter, std::string {}, data, DEFAULT_CACHE_SIZE, py::none() + ); + const py::object next = py::cast(iter).attr("__next__"); + while (true) { + py::object message; + try { + message = next(); + } catch (py::error_already_set& stop) { + if (stop.matches(PyExc_StopIteration)) { + break; + } + throw; + } + callback(message); + } }, - py::arg("itch_file"), - "Read and parse all messages from a binary ITCH file path." + py::arg("data"), + py::arg("callback") ); - // --- Book engine --------------------------------------------------------- + // Book engine (native extras, not part of upstream `itch`) py::class_(mod, "Bbo") .def_readonly("has_bid", &itch::book::Bbo::has_bid) .def_readonly("has_ask", &itch::book::Bbo::has_ask) .def_readonly("bid_shares", &itch::book::Bbo::bid_shares) .def_readonly("ask_shares", &itch::book::Bbo::ask_shares) .def_property_readonly( - "bid_price", [](const itch::book::Bbo& b) { return b.bid_price.to_double(); } + "bid_price", [](const itch::book::Bbo& bbo) { return bbo.bid_price.to_double(); } ) - .def_property_readonly("ask_price", [](const itch::book::Bbo& b) { - return b.ask_price.to_double(); + .def_property_readonly("ask_price", [](const itch::book::Bbo& bbo) { + return bbo.ask_price.to_double(); }); py::class_(mod, "L3Book") @@ -383,7 +846,7 @@ PYBIND11_MODULE(itchcpp, mod) { py::return_value_policy::reference_internal ); - // --- Analytics ----------------------------------------------------------- + // Analytics (native extra) py::class_(mod, "Vwap") .def(py::init<>()) .def( From 1aee4538c293706dfe57a170fe4159840f9bcea4 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:36 +0100 Subject: [PATCH 059/144] python: add itchcpp pure-python package wrapping native extension --- python/itchcpp/__init__.py | 101 +++++++++++++++++ python/itchcpp/_bases.py | 116 +++++++++++++++++++ python/itchcpp/_itchcpp.pyi | 188 +++++++++++++++++++++++++++++++ python/itchcpp/analytics.py | 9 ++ python/itchcpp/book.py | 9 ++ python/itchcpp/indicators.py | 213 +++++++++++++++++++++++++++++++++++ python/itchcpp/messages.py | 102 +++++++++++++++++ python/itchcpp/parser.py | 12 ++ python/itchcpp/py.typed | 0 9 files changed, 750 insertions(+) create mode 100644 python/itchcpp/__init__.py create mode 100644 python/itchcpp/_bases.py create mode 100644 python/itchcpp/_itchcpp.pyi create mode 100644 python/itchcpp/analytics.py create mode 100644 python/itchcpp/book.py create mode 100644 python/itchcpp/indicators.py create mode 100644 python/itchcpp/messages.py create mode 100644 python/itchcpp/parser.py create mode 100644 python/itchcpp/py.typed diff --git a/python/itchcpp/__init__.py b/python/itchcpp/__init__.py new file mode 100644 index 0000000..7e6ba6a --- /dev/null +++ b/python/itchcpp/__init__.py @@ -0,0 +1,101 @@ +"""itchcpp: a native (C++) NASDAQ TotalView-ITCH 5.0 backend. + +``itchcpp`` is a fast, drop-in backend for the pure-Python ``itch`` (PyPI: +``itchfeed``) package. It mirrors that package's public API so existing code only +needs to change the import root, e.g.:: + + from itch.parser import MessageParser # pure-Python + from itchcpp.parser import MessageParser # native backend + +The same message classes, ``MessageParser`` semantics, ``MarketMessage`` helpers +(``decode`` / ``decode_price`` / ``to_bytes`` / ``set_timestamp`` / +``split_timestamp`` / ``get_attributes``), the ``messages`` registry, the +``AllMessages`` constant, ``create_message``, and the :mod:`itchcpp.indicators` +tables are provided. The :mod:`itchcpp.book` and :mod:`itchcpp.analytics` submodules +add native extras (order-book reconstruction and VWAP) that are not part of the +pure-Python API. +""" + +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version + +from . import analytics, book, indicators, messages, parser +from .messages import ( + AddOrderMessage, + AddOrderMPIDAttribution, + AddOrderNoMPIAttributionMessage, + AllMessages, + BrokenTradeMessage, + CrossTradeMessage, + DLCRMessage, + IPOQuotingPeriodUpdateMessage, + LULDAuctionCollarMessage, + MarketMessage, + MarketParticipantPositionMessage, + ModifyOrderMessage, + MWCBDeclineLeveMessage, + MWCBStatusMessage, + NOIIMessage, + NonCrossTradeMessage, + OperationalHaltMessage, + OrderCancelMessage, + OrderDeleteMessage, + OrderExecutedMessage, + OrderExecutedWithPriceMessage, + OrderReplaceMessage, + RegSHOMessage, + RetailPriceImprovementIndicator, + StockDirectoryMessage, + StockTradingActionMessage, + SystemEventMessage, + TradeMessage, + create_message, + messages as message_registry, +) +from .parser import MessageParser + +try: + __version__ = version("itchcpp") +except PackageNotFoundError: # pragma: no cover - source checkout without install + __version__ = "unknown" + +__all__ = [ + "AddOrderMPIDAttribution", + "AddOrderMessage", + "AddOrderNoMPIAttributionMessage", + "AllMessages", + "BrokenTradeMessage", + "CrossTradeMessage", + "DLCRMessage", + "IPOQuotingPeriodUpdateMessage", + "LULDAuctionCollarMessage", + "MWCBDeclineLeveMessage", + "MWCBStatusMessage", + "MarketMessage", + "MarketParticipantPositionMessage", + "MessageParser", + "ModifyOrderMessage", + "NOIIMessage", + "NonCrossTradeMessage", + "OperationalHaltMessage", + "OrderCancelMessage", + "OrderDeleteMessage", + "OrderExecutedMessage", + "OrderExecutedWithPriceMessage", + "OrderReplaceMessage", + "RegSHOMessage", + "RetailPriceImprovementIndicator", + "StockDirectoryMessage", + "StockTradingActionMessage", + "SystemEventMessage", + "TradeMessage", + "__version__", + "analytics", + "book", + "create_message", + "indicators", + "message_registry", + "messages", + "parser", +] diff --git a/python/itchcpp/_bases.py b/python/itchcpp/_bases.py new file mode 100644 index 0000000..f9e0a94 --- /dev/null +++ b/python/itchcpp/_bases.py @@ -0,0 +1,116 @@ +"""Abstract base classes for the ITCH message hierarchy. + +These mirror the bases in ``itch.messages`` from the pure-Python ``itchfeed`` +package. The concrete message classes are implemented in the compiled ``_itchcpp`` +extension and registered as virtual subclasses of these bases (so ``isinstance`` +checks behave as upstream), which is why the bases live in their own dependency-free +module: it lets the type stubs reference them without an import cycle. + +The method bodies match the upstream :class:`MarketMessage` and are used only when a +base is instantiated directly; the native classes carry their own C++ implementations. +""" + +from __future__ import annotations + +from abc import ABCMeta +from dataclasses import make_dataclass +from typing import TYPE_CHECKING, Any, List, Tuple, overload + + +class MarketMessage(metaclass=ABCMeta): + """Abstract base for every ITCH message.""" + + message_type: bytes + description: str + message_size: int + price_precision: int = 4 + timestamp: int + stock_locate: int + tracking_number: int + + if TYPE_CHECKING: + # The concrete (native) subclasses are default-constructible and can also + # be built from a raw message body; these overloads are for type-checkers + # only and add no runtime __init__ to the abstract base. + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, body: bytes) -> None: ... + + def __repr__(self) -> str: + return repr(self.decode()) + + def __bytes__(self) -> bytes: + return self.to_bytes() + + def to_bytes(self) -> bytes: # pragma: no cover - overridden by concrete classes + raise NotImplementedError + + def set_timestamp(self, ts1: int, ts2: int) -> None: + self.timestamp = ts2 | (ts1 << 32) + + def split_timestamp(self) -> Tuple[int, int]: + ts1 = self.timestamp >> 32 + ts2 = self.timestamp - (ts1 << 32) + return (ts1, ts2) + + def decode_price(self, price_attr: str) -> float: + price = getattr(self, price_attr) + if callable(price): + raise ValueError(f"Please check the price attribute for {price_attr}") + return price / (10 ** getattr(self, "price_precision")) + + def decode(self, prefix: str = "") -> Any: + builtin_attrs = {} + for attr in dir(self): + if attr.startswith("__"): + continue + value = getattr(self, attr) + if "price" in attr and attr not in ("price_precision", "decode_price"): + value = self.decode_price(attr) + if callable(value): + continue + if isinstance(value, (int, float, str, bool)): + builtin_attrs[attr] = value + elif isinstance(value, bytes): + try: + builtin_attrs[attr] = value.decode(encoding="ascii").rstrip() + except UnicodeDecodeError: + builtin_attrs[attr] = value + fields = [(key, type(value)) for key, value in builtin_attrs.items()] + decoded_class = make_dataclass(f"{prefix}{self.__class__.__name__}", fields) + return decoded_class(**builtin_attrs) + + def get_attributes(self, call_able: bool = False) -> List[str]: + attrs = [attr for attr in dir(self) if not attr.startswith("__")] + return [attr for attr in attrs if callable(getattr(self, attr)) == call_able] + + +class AddOrderMessage(MarketMessage, metaclass=ABCMeta): + """Abstract base for the two Add Order message variants.""" + + order_reference_number: int + buy_sell_indicator: bytes + shares: int + stock: bytes + price: int + + +class ModifyOrderMessage(MarketMessage, metaclass=ABCMeta): + """Abstract base for messages that modify a resting order.""" + + order_reference_number: int + + +class TradeMessage(MarketMessage, metaclass=ABCMeta): + """Abstract base for trade messages.""" + + match_number: int + + +__all__ = [ + "AddOrderMessage", + "MarketMessage", + "ModifyOrderMessage", + "TradeMessage", +] diff --git a/python/itchcpp/_itchcpp.pyi b/python/itchcpp/_itchcpp.pyi new file mode 100644 index 0000000..de87da6 --- /dev/null +++ b/python/itchcpp/_itchcpp.pyi @@ -0,0 +1,188 @@ +"""Type stubs for the compiled ``itchcpp._itchcpp`` extension module. + +The concrete message classes are implemented in C++. For typing they are presented +as subclasses of the abstract bases in :mod:`itchcpp._bases` (``MarketMessage`` and +friends), matching the runtime ``isinstance`` behavior established by virtual-subclass +registration in :mod:`itchcpp.messages`. +""" + +from typing import Any, BinaryIO, Callable, Iterator + +from ._bases import ( + AddOrderMessage, + MarketMessage, + ModifyOrderMessage, + TradeMessage, +) + +AllMessages: bytes +messages: dict[bytes, type[MarketMessage]] + +class SystemEventMessage(MarketMessage): + event_code: bytes + +class StockDirectoryMessage(MarketMessage): + stock: bytes + market_category: bytes + financial_status_indicator: bytes + round_lot_size: int + round_lots_only: bytes + issue_classification: bytes + issue_sub_type: bytes + authenticity: bytes + short_sale_threshold_indicator: bytes + ipo_flag: bytes + luld_ref: bytes + etp_flag: bytes + etp_leverage_factor: int + inverse_indicator: bytes + +class StockTradingActionMessage(MarketMessage): + stock: bytes + trading_state: bytes + reserved: bytes + reason: bytes + +class RegSHOMessage(MarketMessage): + stock: bytes + reg_sho_action: bytes + +class MarketParticipantPositionMessage(MarketMessage): + mpid: bytes + stock: bytes + primary_market_maker: bytes + market_maker_mode: bytes + market_participant_state: bytes + +class MWCBDeclineLeveMessage(MarketMessage): + level1_price: int + level2_price: int + level3_price: int + +class MWCBStatusMessage(MarketMessage): + breached_level: bytes + +class IPOQuotingPeriodUpdateMessage(MarketMessage): + stock: bytes + ipo_release_time: int + ipo_release_qualifier: bytes + ipo_price: int + +class LULDAuctionCollarMessage(MarketMessage): + stock: bytes + auction_collar_reference_price: int + upper_auction_collar_price: int + lower_auction_collar_price: int + auction_collar_extention: int + +class OperationalHaltMessage(MarketMessage): + stock: bytes + market_code: bytes + operational_halt_action: bytes + +class AddOrderNoMPIAttributionMessage(AddOrderMessage): ... + +class AddOrderMPIDAttribution(AddOrderMessage): + attribution: bytes + +class OrderExecutedMessage(ModifyOrderMessage): + executed_shares: int + match_number: int + +class OrderExecutedWithPriceMessage(ModifyOrderMessage): + executed_shares: int + match_number: int + printable: bytes + execution_price: int + +class OrderCancelMessage(ModifyOrderMessage): + cancelled_shares: int + +class OrderDeleteMessage(ModifyOrderMessage): ... + +class OrderReplaceMessage(ModifyOrderMessage): + new_order_reference_number: int + shares: int + price: int + +class NonCrossTradeMessage(TradeMessage): + order_reference_number: int + buy_sell_indicator: bytes + shares: int + stock: bytes + price: int + +class CrossTradeMessage(TradeMessage): + shares: int + stock: bytes + cross_price: int + cross_type: bytes + +class BrokenTradeMessage(TradeMessage): ... + +class NOIIMessage(MarketMessage): + paired_shares: int + imbalance_shares: int + imbalance_direction: bytes + stock: bytes + far_price: int + near_price: int + current_reference_price: int + cross_type: bytes + variation_indicator: bytes + +class RetailPriceImprovementIndicator(MarketMessage): + stock: bytes + interest_flag: bytes + +class DLCRMessage(MarketMessage): + stock: bytes + open_eligibility_status: bytes + minimum_allowable_price: int + maximum_allowable_price: int + near_execution_price: int + near_execution_time: int + lower_price_range_collar: int + upper_price_range_collar: int + +def create_message(message_type: bytes, **kwargs: Any) -> MarketMessage: ... + +class MessageParser: + message_type: bytes + def __init__(self, message_type: bytes = ...) -> None: ... + def get_message_type(self, message: bytes) -> MarketMessage: ... + def parse_stream( + self, data: bytes, save_file: BinaryIO | None = ... + ) -> Iterator[MarketMessage]: ... + def parse_file( + self, file: BinaryIO, cachesize: int = ..., save_file: BinaryIO | None = ... + ) -> Iterator[MarketMessage]: ... + def parse_messages( + self, data: bytes | BinaryIO, callback: Callable[[MarketMessage], None] + ) -> None: ... + +class Bbo: + has_bid: bool + has_ask: bool + bid_shares: int + ask_shares: int + bid_price: float + ask_price: float + +class L3Book: + symbol: str + def bbo(self) -> Bbo: ... + +class BookManager: + def __init__(self) -> None: ... + def track_symbol(self, symbol: str) -> None: ... + def process_stream(self, raw_binary_data: bytes) -> None: ... + def book_count(self) -> int: ... + def book_for_symbol(self, symbol: str) -> L3Book: ... + +class Vwap: + def __init__(self) -> None: ... + def add(self, raw_price: int, shares: int) -> None: ... + def value(self) -> float: ... + def volume(self) -> int: ... + def reset(self) -> None: ... diff --git a/python/itchcpp/analytics.py b/python/itchcpp/analytics.py new file mode 100644 index 0000000..835c531 --- /dev/null +++ b/python/itchcpp/analytics.py @@ -0,0 +1,9 @@ +"""Native streaming analytics (an itchcpp extra, beyond the pure-Python ``itch`` API). + +Currently exposes a volume-weighted average price (VWAP) accumulator fed with raw +4-decimal prices and share quantities. +""" + +from ._itchcpp import Vwap + +__all__ = ["Vwap"] diff --git a/python/itchcpp/book.py b/python/itchcpp/book.py new file mode 100644 index 0000000..4d3b1a2 --- /dev/null +++ b/python/itchcpp/book.py @@ -0,0 +1,9 @@ +"""Native order-book engine (an itchcpp extra, beyond the pure-Python ``itch`` API). + +Reconstructs per-symbol L3 books and best bid/offer directly from an ITCH stream +in a single C++ pass. +""" + +from ._itchcpp import Bbo, BookManager, L3Book + +__all__ = ["Bbo", "BookManager", "L3Book"] diff --git a/python/itchcpp/indicators.py b/python/itchcpp/indicators.py new file mode 100644 index 0000000..852e5bd --- /dev/null +++ b/python/itchcpp/indicators.py @@ -0,0 +1,213 @@ +"""Indicator/code lookup tables for ITCH 5.0 fields. + +Vendored verbatim from ``itch.indicators`` in the pure-Python ``itchfeed`` package +so that code reading these maps works unchanged against the native backend. Each +table maps a single- or multi-byte code to its human-readable description. +""" + +SYSTEM_EVENT_CODES = { + b"O": "Start of Messages", + b"S": "Start of System hours", + b"Q": "Start of Market hours", + b"M": "End of Market hours", + b"E": "End of System hours", + b"C": "End of Messages", +} + + +MARKET_CATEGORY = { + b"N": "NYSE", + b"A": "AMEX", + b"P": "Arca", + b"Q": "NASDAQ Global Select", + b"G": "NASDAQ Global Market", + b"S": "NASDAQ Capital Market", + b"Z": "BATS", + b"V": "Investors Exchange", + b" ": "Not available", +} + + +FINANCIAL_STATUS_INDICATOR = { + b"D": "Deficient", + b"E": "Delinquent", + b"Q": "Bankrupt", + b"S": "Suspended", + b"G": "Deficient and Bankrupt", + b"H": "Deficient and Delinquent", + b"J": "Delinquent and Bankrupt", + b"K": "Deficient, Delinquent and Bankrupt", + b"C": "Creations and/or Redemptions Suspended for Exchange Traded Product", + b"N": "Normal (Default): Issuer is NOT Deficient, Delinquent, or Bankrupt", + b" ": "Not available. Firms should refer to SIAC feeds for code if needed", +} + + +ISSUE_CLASSIFICATION_VALUES = { + b"A": "American Depositary Share", + b"B": "Bond", + b"C": "Common Stock", + b"F": "Depository Receipt", + b"I": "144A", + b"L": "Limited Partnership", + b"N": "Notes", + b"O": "Ordinary Share", + b"P": "Preferred Stock", + b"Q": "Other Securities", + b"R": "Right", + b"S": "Shares of Beneficial Interest", + b"T": "Convertible Debenture", + b"U": "Unit", + b"V": "Units/Beneficial Interest", + b"W": "Warrant", +} + + +ISSUE_SUB_TYPE_VALUES = { + b"A": "Preferred Trust Securities", + b"AI": "Alpha Index ETNs", + b"B": "Index Based Derivative", + b"C": "Common Shares", + b"CB": "Commodity Based Trust Shares", + b"CF": "Commodity Futures Trust Shares", + b"CL": "Commodity-Linked Securities", + b"CM": "Commodity Index Trust Shares", + b"CO": "Collateralized Mortgage Obligation", + b"CT": "Currency Trust Shares", + b"CU": "Commodity-Currency-Linked Securities", + b"CW": "Currency Warrants", + b"D": "Global Depositary Shares", + b"E": "ETF-Portfolio Depositary Receipt", + b"EG": "Equity Gold Shares", + b"EI": "ETN-Equity Index-Linked Securities", + b"EM": "NextShares Exchange Traded Managed Fund*", + b"EN": "Exchange Traded Notes", + b"EU": "Equity Units", + b"F": "HOLDRS", + b"FI": "ETN-Fixed Income-Linked Securities", + b"FL": "ETN-Futures-Linked Securities", + b"G": "Global Shares", + b"I": "ETF-Index Fund Shares", + b"IR": "Interest Rate", + b"IW": "Index Warrant", + b"IX": "Index-Linked Exchangeable Notes", + b"J": "Corporate Backed Trust Security", + b"L": "Contingent Litigation Right", + b"LL": "Limited Liability Company (LLC)", + b"M": "Equity-Based Derivative", + b"MF": "Managed Fund Shares", + b"ML": "ETN-Multi-Factor Index-Linked Securities", + b"MT": "Managed Trust Securities", + b"N": "NY Registry Shares", + b"O": "Open Ended Mutual Fund", + b"P": "Privately Held Security", + b"PP": "Poison Pill", + b"PU": "Partnership Units", + b"Q": "Closed-End Funds", + b"R": "Reg-S", + b"RC": "Commodity-Redeemable Commodity-Linked Securities", + b"RF": "ETN-Redeemable Futures-Linked Securities", + b"RT": "REIT", + b"RU": "Commodity-Redeemable Currency-Linked Securities", + b"S": "SEED", + b"SC": "Spot Rate Closing", + b"SI": "Spot Rate Intraday", + b"T": "Tracking Stock", + b"TC": "Trust Certificates", + b"TU": "Trust Units", + b"U": "Portal", + b"V": "Contingent Value Right", + b"W": "Trust Issued Receipts", + b"WC": "World Currency Option", + b"X": "Trust", + b"Y": "Other", + b"Z": "Not Applicable", +} + + +TRADING_STATES = { + b"H": "Halted across all U.S. equity markets / SROs", + b"P": "Paused across all U.S. equity markets / SROs", + b"Q": "Quotation only period for cross-SRO halt or pause", + b"T": "Trading on NASDAQ", +} + + +TRADING_ACTION_REASON_CODES = { + b"T1": "Halt News Pending", + b"T2": "Halt News Disseminated", + b"T3": "News and Resumption Times", + b"T5": "Single Security Trading Pause In Effect", + b"T6": "Regulatory Halt - Extraordinary Market Activity", + b"T7": "Single Security Trading Pause / Quotation Only Period", + b"T8": "Halt ETF", + b"T12": "Trading Halted; For Information Requested by Listing Market", + b"H4": "Halt Non-Compliance", + b"H9": "Halt Filings Not Current", + b"H10": "Halt SEC Trading Suspension", + b"H11": "Halt Regulatory Concern", + b"O1": "Operations Halt; Contact Market Operations", + b"LUDP": "Volatility Trading Pause", + b"LUDS": "Volatility Trading Pause - Straddle Condition", + b"MWC0": "Market Wide Circuit Breaker Halt - Carry over from previous day", + b"MWC1": "Market Wide Circuit Breaker Halt - Level 1", + b"MWC2": "Market Wide Circuit Breaker Halt - Level 2", + b"MWC3": "Market Wide Circuit Breaker Halt - Level 3", + b"MWCQ": "Market Wide Circuit Breaker Resumption", + b"IPO1": "IPO Issue Not Yet Trading", + b"IPOQ": "IPO Security Released for Quotation (Nasdaq Securities Only)", + b"IPOE": "IPO Security — Positioning Window Extension (Nasdaq Securities Only)", + b"M1": "Corporate Action", + b"M2": "Quotation Not Available", + b"R1": "New Issue Available", + b"R2": "Issue Available", + b"R4": "Qualifications Issues Reviewed/Resolved; Quotations/Trading to Resume", + b"R9": "Filing Requirements Satisfied/Resolved; Quotations/Trading To Resume", + b"C3": "Issuer News Not Forthcoming; Quotations/Trading To Resume", + b"C4": "Qualifications Halt Ended; Maintenance Requirements Met; Resume", + b"C9": "Qualifications Halt Concluded; Filings Met; Quotes/Trades To Resume", + b"C11": "Trade Halt Concluded By Other Regulatory Authority; Quotes/Trades Resume", + b" ": "Reason Not Available", +} + + +PRIMARY_MARKET_MAKER = { + b"Y": "Primary market maker", + b"N": "Non-primary market maker", +} + + +MARKET_MAKER_MODE = { + b"N": "Normal", + b"P": "Passive", + b"S": "Syndicate", + b"R": "Pre-syndicate", + b"L": "Penalty", +} + + +MARKET_PARTICIPANT_STATE = { + b"A": "Active", + b"E": "Excused", + b"W": "Withdrawn", + b"S": "Suspended", + b"D": "Deleted", +} + + +PRICE_VARIATION_INDICATOR = { + b"L": "Less than 1%", + b"1": "1 to 1.99%", + b"2": "2 to 2.99%", + b"3": "3 to 3.99%", + b"4": "4 to 4.99%", + b"5": "5 to 5.99%", + b"6": "6 to 6.99%", + b"7": "7 to 7.99%", + b"8": "8 to 8.99%", + b"9": "9 to 9.99%", + b"A": "10 to 19.99%", + b"B": "20 to 29.99%", + b"C": "30% or greater", + b" ": "Cannot be calculated", +} diff --git a/python/itchcpp/messages.py b/python/itchcpp/messages.py new file mode 100644 index 0000000..a9d628f --- /dev/null +++ b/python/itchcpp/messages.py @@ -0,0 +1,102 @@ +"""Message classes for the native ITCH backend. + +Mirrors ``itch.messages`` from the pure-Python ``itchfeed`` package: the same +concrete message classes (re-exported from the compiled ``_itchcpp`` extension), +the ``messages`` registry, the ``AllMessages`` constant, and the ``create_message`` +factory. The abstract base classes (:class:`MarketMessage`, :class:`AddOrderMessage`, +:class:`ModifyOrderMessage`, :class:`TradeMessage`) come from :mod:`itchcpp._bases` +and the native classes are registered here as virtual subclasses so ``isinstance`` +checks behave exactly as they do upstream. +""" + +from __future__ import annotations + +from ._bases import ( + AddOrderMessage, + MarketMessage, + ModifyOrderMessage, + TradeMessage, +) +from ._itchcpp import ( # noqa: F401 (re-exported) + AddOrderMPIDAttribution, + AddOrderNoMPIAttributionMessage, + AllMessages, + BrokenTradeMessage, + CrossTradeMessage, + DLCRMessage, + IPOQuotingPeriodUpdateMessage, + LULDAuctionCollarMessage, + MarketParticipantPositionMessage, + MWCBDeclineLeveMessage, + MWCBStatusMessage, + NOIIMessage, + NonCrossTradeMessage, + OperationalHaltMessage, + OrderCancelMessage, + OrderDeleteMessage, + OrderExecutedMessage, + OrderExecutedWithPriceMessage, + OrderReplaceMessage, + RegSHOMessage, + RetailPriceImprovementIndicator, + StockDirectoryMessage, + StockTradingActionMessage, + SystemEventMessage, + create_message, + messages, +) + + +# Register the native classes as virtual subclasses for isinstance/issubclass parity. +_ADD_ORDER = (AddOrderNoMPIAttributionMessage, AddOrderMPIDAttribution) +_MODIFY_ORDER = ( + OrderExecutedMessage, + OrderExecutedWithPriceMessage, + OrderCancelMessage, + OrderDeleteMessage, + OrderReplaceMessage, +) +_TRADE = (NonCrossTradeMessage, CrossTradeMessage, BrokenTradeMessage) + +for _cls in messages.values(): + MarketMessage.register(_cls) +for _cls in _ADD_ORDER: + AddOrderMessage.register(_cls) +for _cls in _MODIFY_ORDER: + ModifyOrderMessage.register(_cls) +for _cls in _TRADE: + TradeMessage.register(_cls) + + +__all__ = [ + "AddOrderMessage", + "AddOrderMPIDAttribution", + "AddOrderNoMPIAttributionMessage", + "AllMessages", + "BrokenTradeMessage", + "CrossTradeMessage", + "DLCRMessage", + "IPOQuotingPeriodUpdateMessage", + "LULDAuctionCollarMessage", + "MarketMessage", + "MarketParticipantPositionMessage", + "ModifyOrderMessage", + "MWCBDeclineLeveMessage", + "MWCBStatusMessage", + "NOIIMessage", + "NonCrossTradeMessage", + "OperationalHaltMessage", + "OrderCancelMessage", + "OrderDeleteMessage", + "OrderExecutedMessage", + "OrderExecutedWithPriceMessage", + "OrderReplaceMessage", + "RegSHOMessage", + "RetailPriceImprovementIndicator", + "StockDirectoryMessage", + "StockTradingActionMessage", + "SystemEventMessage", + "TradeMessage", + "create_message", + "messages", +] diff --git a/python/itchcpp/parser.py b/python/itchcpp/parser.py new file mode 100644 index 0000000..1f1b62e --- /dev/null +++ b/python/itchcpp/parser.py @@ -0,0 +1,12 @@ +"""Native ITCH message parser. + +Mirrors ``itch.parser`` from the pure-Python ``itchfeed`` package. The +:class:`MessageParser` is implemented in C++ (the ``_itchcpp`` extension) and +exposes the same interface: an optional ``message_type`` byte filter, lazy +``parse_stream`` / ``parse_file`` iterators, ``parse_messages`` callback dispatch, +and ``get_message_type``. +""" + +from ._itchcpp import AllMessages, MessageParser + +__all__ = ["AllMessages", "MessageParser"] diff --git a/python/itchcpp/py.typed b/python/itchcpp/py.typed new file mode 100644 index 0000000..e69de29 From 5c1763e713352406f8c80e416498a31a99972e2f Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:42 +0100 Subject: [PATCH 060/144] test: vendor itchfeed sample message fixtures --- python/tests/data.py | 230 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 230 insertions(+) create mode 100644 python/tests/data.py diff --git a/python/tests/data.py b/python/tests/data.py new file mode 100644 index 0000000..ea61eb6 --- /dev/null +++ b/python/tests/data.py @@ -0,0 +1,230 @@ +"""Sample message payloads for every ITCH 5.0 message type. + +Vendored from the upstream ``itchfeed`` test suite so the native backend can be +exercised against the same fixtures. +""" + +DEFAULT_STOCK_LOCATE = 1 +DEFAULT_TRACKING_NUMBER = 2 +DEFAULT_TIMESTAMP = 1651500000 * 1_000_000_000 +DEFAULT_ORDER_REF = 1234567890 +DEFAULT_MATCH_NUM = 9876543210 +DEFAULT_STOCK = b"AAPL " +DEFAULT_MPID = b"NSDQ" + +TEST_DATA = { + b"S": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "event_code": b"O", + }, + b"R": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "market_category": b"Q", + "financial_status_indicator": b"N", + "round_lot_size": 100, + "round_lots_only": b"N", + "issue_classification": b"C", + "issue_sub_type": b"I ", + "authenticity": b"P", + "short_sale_threshold_indicator": b"N", + "ipo_flag": b"N", + "luld_ref": b"1", + "etp_flag": b"N", + "etp_leverage_factor": 0, + "inverse_indicator": b"N", + }, + b"H": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "trading_state": b"T", + "reserved": b"\x00", + "reason": b"T1 ", + }, + b"Y": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "reg_sho_action": b"0", + }, + b"L": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "mpid": DEFAULT_MPID, + "stock": DEFAULT_STOCK, + "primary_market_maker": b"Y", + "market_maker_mode": b"N", + "market_participant_state": b"A", + }, + b"V": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "level1_price": 10000000000, + "level2_price": 20000000000, + "level3_price": 30000000000, + }, + b"W": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "breached_level": b"1", + }, + b"K": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "ipo_release_time": 34200, + "ipo_release_qualifier": b"A", + "ipo_price": 255000, + }, + b"J": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "auction_collar_reference_price": 1500000, + "upper_auction_collar_price": 1600000, + "lower_auction_collar_price": 1400000, + "auction_collar_extention": 1, + }, + b"h": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "market_code": b"Q", + "operational_halt_action": b"H", + }, + b"A": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "buy_sell_indicator": b"B", + "shares": 100, + "stock": DEFAULT_STOCK, + "price": 1501234, + }, + b"F": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "buy_sell_indicator": b"S", + "shares": 200, + "stock": DEFAULT_STOCK, + "price": 1505000, + "attribution": DEFAULT_MPID, + }, + b"E": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "executed_shares": 50, + "match_number": DEFAULT_MATCH_NUM, + }, + b"C": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "executed_shares": 50, + "match_number": DEFAULT_MATCH_NUM, + "printable": b"Y", + "execution_price": 1502000, + }, + b"X": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "cancelled_shares": 100, + }, + b"D": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + }, + b"U": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "new_order_reference_number": DEFAULT_ORDER_REF + 1, + "shares": 150, + "price": 1510000, + }, + b"P": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "buy_sell_indicator": b"B", + "shares": 100, + "stock": DEFAULT_STOCK, + "price": 1503000, + "match_number": DEFAULT_MATCH_NUM, + }, + b"Q": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "shares": 100000, + "stock": DEFAULT_STOCK, + "cross_price": 1500000, + "match_number": DEFAULT_MATCH_NUM, + "cross_type": b"O", + }, + b"B": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "match_number": DEFAULT_MATCH_NUM, + }, + b"I": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "paired_shares": 50000, + "imbalance_shares": 10000, + "imbalance_direction": b"B", + "stock": DEFAULT_STOCK, + "far_price": 1520000, + "near_price": 1510000, + "current_reference_price": 1500000, + "cross_type": b"O", + "variation_indicator": b" ", + }, + b"N": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "interest_flag": b"A", + }, + b"O": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "open_eligibility_status": b"Y", + "minimum_allowable_price": 1800000, + "maximum_allowable_price": 2200000, + "near_execution_price": 2000000, + "near_execution_time": DEFAULT_TIMESTAMP, + "lower_price_range_collar": 1900000, + "upper_price_range_collar": 2100000, + }, +} From dc4e996d9418bf4d5988dcb7214e5e8f3efafcf1 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:45 +0100 Subject: [PATCH 061/144] test: update binding smoke tests for itchcpp package layout --- python/tests/test_bindings.py | 36 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py index 6377fb1..a90f791 100644 --- a/python/tests/test_bindings.py +++ b/python/tests/test_bindings.py @@ -1,20 +1,23 @@ -"""Tests for the native ``itchcpp`` bindings. +"""Smoke tests for the native ``itchcpp`` bindings. -These mirror the pure-Python ``itch`` package's API so the native module can be a -drop-in backend: a ``MessageParser`` producing typed message objects with -``decode_price`` semantics, plus the book engine and analytics. +Covers the top-level package surface (parser, raw attribute semantics) and the +native extras (the order-book engine and the VWAP accumulator) that go beyond the +pure-Python ``itch`` API. """ import struct import itchcpp +from itchcpp.parser import MessageParser def _frame(payload: bytes) -> bytes: return struct.pack(">H", len(payload)) + payload -def _add_order(locate: int, ref: int, side: bytes, shares: int, sym: bytes, price: int) -> bytes: +def _add_order( + locate: int, ref: int, side: bytes, shares: int, sym: bytes, price: int +) -> bytes: payload = ( b"A" + struct.pack(">HH", locate, 0) @@ -28,39 +31,40 @@ def _add_order(locate: int, ref: int, side: bytes, shares: int, sym: bytes, pric return _frame(payload) -def test_parse_stream_yields_typed_messages() -> None: +def test_parse_stream_yields_typed_messages_with_raw_attributes() -> None: buffer = _add_order(7, 42, b"B", 500, b"AAPL", 1_500_000) - parser = itchcpp.MessageParser() - messages = parser.parse_stream(buffer) + messages = list(MessageParser().parse_stream(buffer)) assert len(messages) == 1 message = messages[0] - assert isinstance(message, itchcpp.AddOrderMessage) - assert message.message_type == "A" - assert message.stock == "AAPL" + assert isinstance(message, itchcpp.AddOrderNoMPIAttributionMessage) + # Raw attributes are bytes/int, exactly like the pure-Python package. + assert message.message_type == b"A" + assert message.stock == b"AAPL " assert message.shares == 500 assert message.order_reference_number == 42 - # decode_price applies the 4-decimal scale, matching the upstream API. + assert message.price == 1_500_000 + # decode_price applies the 4-decimal scale; decode() trims the symbol. assert message.decode_price("price") == 150.0 + assert message.decode().stock == "AAPL" def test_book_manager_reconstructs_bbo() -> None: buffer = _add_order(7, 1, b"B", 300, b"AAPL", 1_500_000) + _add_order( 7, 2, b"S", 100, b"AAPL", 1_500_200 ) - manager = itchcpp.BookManager() + manager = itchcpp.book.BookManager() manager.process_stream(buffer) assert manager.book_count() == 1 - book = manager.book_for_symbol("AAPL") - bbo = book.bbo() + bbo = manager.book_for_symbol("AAPL").bbo() assert bbo.bid_price == 150.0 assert bbo.ask_price == 150.02 assert bbo.bid_shares == 300 def test_vwap_accumulates() -> None: - vwap = itchcpp.Vwap() + vwap = itchcpp.analytics.Vwap() vwap.add(100, 10) vwap.add(200, 30) assert vwap.volume() == 40 From e7a6546a1eb5c1f2e58d2a796fcbd06bfc832f6b Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 11:40:49 +0100 Subject: [PATCH 062/144] test: add itchfeed behavioral parity tests --- python/tests/test_parity.py | 206 ++++++++++++++++++++++++++++++++++++ 1 file changed, 206 insertions(+) create mode 100644 python/tests/test_parity.py diff --git a/python/tests/test_parity.py b/python/tests/test_parity.py new file mode 100644 index 0000000..5585943 --- /dev/null +++ b/python/tests/test_parity.py @@ -0,0 +1,206 @@ +"""Behavioral parity tests for the native ``itchcpp`` backend. + +These mirror the upstream ``itchfeed`` test suite, exercised through the public +API (``create_message`` -> ``to_bytes`` -> frame -> ``parse_stream``/``parse_file``, +plus ``decode`` / ``decode_price`` / ``get_attributes``) so the native module is +verified to behave as a drop-in replacement. +""" + +from __future__ import annotations + +from dataclasses import is_dataclass +from io import BytesIO + +import pytest + +import itchcpp.messages as msgs +from data import TEST_DATA +from itchcpp.messages import create_message, messages +from itchcpp.parser import MessageParser + + +def _frame(body: bytes) -> bytes: + """Wrap a message body in the ITCH framing the parser expects.""" + return b"\x00" + len(body).to_bytes(1, "big") + body + + +# Parser behavior + + +def test_parse_stream_single_message() -> None: + body = create_message(b"S", **TEST_DATA[b"S"]).to_bytes() + messages_out = list(MessageParser().parse_stream(_frame(body))) + assert len(messages_out) == 1 + assert isinstance(messages_out[0], msgs.SystemEventMessage) + + +def test_parse_stream_multiple_messages() -> None: + data = _frame(create_message(b"S", **TEST_DATA[b"S"]).to_bytes()) + _frame( + create_message(b"A", **TEST_DATA[b"A"]).to_bytes() + ) + out = list(MessageParser().parse_stream(data)) + assert len(out) == 2 + assert isinstance(out[0], msgs.SystemEventMessage) + assert isinstance(out[1], msgs.AddOrderNoMPIAttributionMessage) + + +def test_parse_file_reads_binary_object() -> None: + body = create_message(b"S", **TEST_DATA[b"S"]).to_bytes() + out = list(MessageParser().parse_file(BytesIO(_frame(body)))) + assert len(out) == 1 + assert isinstance(out[0], msgs.SystemEventMessage) + + +def test_parse_file_streams_in_chunks() -> None: + # Many messages with a tiny cache size forces the iterator to refill its buffer + # across frame boundaries, exercising the lazy chunked-read path. + one = _frame(create_message(b"A", **TEST_DATA[b"A"]).to_bytes()) + blob = one * 50 + out = list(MessageParser().parse_file(BytesIO(blob), cachesize=7)) + assert len(out) == 50 + assert all(isinstance(m, msgs.AddOrderNoMPIAttributionMessage) for m in out) + assert out[0].order_reference_number == TEST_DATA[b"A"]["order_reference_number"] + + +def test_unknown_message_type_raises() -> None: + with pytest.raises(ValueError): + list(MessageParser().parse_stream(b"\x00\x01\x00")) + + +def test_incomplete_message_is_skipped() -> None: + body = create_message(b"S", **TEST_DATA[b"S"]).to_bytes() + data = _frame(body)[:-1] + assert list(MessageParser().parse_stream(data)) == [] + + +def test_stop_on_system_event_c() -> None: + kwargs = dict(TEST_DATA[b"S"], event_code=b"C") + data = _frame(create_message(b"S", **kwargs).to_bytes()) + _frame( + create_message(b"A", **TEST_DATA[b"A"]).to_bytes() + ) + out = list(MessageParser().parse_stream(data)) + assert len(out) == 1 + assert isinstance(out[0], msgs.SystemEventMessage) + + +def test_message_type_filter() -> None: + data = _frame(create_message(b"S", **TEST_DATA[b"S"]).to_bytes()) + _frame( + create_message(b"A", **TEST_DATA[b"A"]).to_bytes() + ) + out = list(MessageParser(message_type=b"A").parse_stream(data)) + assert len(out) == 1 + assert isinstance(out[0], msgs.AddOrderNoMPIAttributionMessage) + + +def test_parse_messages_callback() -> None: + data = _frame(create_message(b"S", **TEST_DATA[b"S"]).to_bytes()) + collected: list = [] + MessageParser().parse_messages(data, collected.append) + assert len(collected) == 1 + assert isinstance(collected[0], msgs.SystemEventMessage) + + +@pytest.mark.parametrize("message_type", list(TEST_DATA.keys())) +def test_parse_all_message_types(message_type: bytes) -> None: + body = create_message(message_type, **TEST_DATA[message_type]).to_bytes() + out = list(MessageParser().parse_stream(_frame(body))) + assert len(out) == 1 + assert isinstance(out[0], messages[message_type]) + + +# create_message / pack / unpack / decode + + +@pytest.mark.parametrize("message_type", list(TEST_DATA.keys())) +def test_create_message_round_trips(message_type: bytes) -> None: + sample = TEST_DATA[message_type] + body = create_message(message_type, **sample).to_bytes() + + assert body.startswith(message_type) + assert len(body) == messages[message_type].message_size + + unpacked = messages[message_type](body) + for key, expected in sample.items(): + if key == "timestamp": + expected &= (1 << 48) - 1 + assert getattr(unpacked, key) == expected, f"{message_type!r}.{key}" + + assert unpacked.to_bytes() == body + + +@pytest.mark.parametrize("message_type", list(TEST_DATA.keys())) +def test_decode_produces_prefixed_dataclass(message_type: bytes) -> None: + sample = TEST_DATA[message_type] + instance = messages[message_type](create_message(message_type, **sample).to_bytes()) + + decoded = instance.decode(prefix="Test") + assert is_dataclass(decoded) + assert decoded.__class__.__name__ == f"Test{messages[message_type].__name__}" + + for key, expected in sample.items(): + decoded_value = getattr(decoded, key) + if key == "timestamp": + assert decoded_value == (expected & ((1 << 48) - 1)) + elif "price" in key and key not in ("price_precision",): + precision = 8 if key.startswith("level") else 4 + assert decoded_value == pytest.approx(expected / (10**precision)) + elif isinstance(expected, bytes): + assert decoded_value == expected.decode("ascii").rstrip() + else: + assert decoded_value == expected + + +def test_get_attributes_partitions_data_and_methods() -> None: + sample = TEST_DATA[b"A"] + instance = messages[b"A"](create_message(b"A", **sample).to_bytes()) + + non_callable = instance.get_attributes(call_able=False) + callable_attrs = instance.get_attributes(call_able=True) + + assert "message_type" in non_callable + assert "description" in non_callable + assert "timestamp" in non_callable + assert "stock" in non_callable + assert "price" in non_callable + + for name in ( + "to_bytes", + "decode", + "decode_price", + "set_timestamp", + "split_timestamp", + ): + assert name in callable_attrs + + assert not set(non_callable) & set(callable_attrs) + for key in sample: + assert key in non_callable + + +def test_decode_price_scales_and_validates() -> None: + add = messages[b"A"](create_message(b"A", **TEST_DATA[b"A"]).to_bytes()) + assert add.decode_price("price") == pytest.approx(TEST_DATA[b"A"]["price"] / 10000) + + mwcb = messages[b"V"](create_message(b"V", **TEST_DATA[b"V"]).to_bytes()) + assert mwcb.decode_price("level1_price") == pytest.approx( + TEST_DATA[b"V"]["level1_price"] / 10**8 + ) + + with pytest.raises( + ValueError, match="Please check the price attribute for to_bytes" + ): + add.decode_price("to_bytes") + with pytest.raises(TypeError): + add.decode_price("stock") + + +def test_isinstance_hierarchy() -> None: + add = messages[b"A"](create_message(b"A", **TEST_DATA[b"A"]).to_bytes()) + trade = messages[b"P"](create_message(b"P", **TEST_DATA[b"P"]).to_bytes()) + execed = messages[b"E"](create_message(b"E", **TEST_DATA[b"E"]).to_bytes()) + + assert isinstance(add, msgs.AddOrderMessage) + assert isinstance(add, msgs.MarketMessage) + assert isinstance(trade, msgs.TradeMessage) + assert isinstance(execed, msgs.ModifyOrderMessage) + assert not isinstance(add, msgs.TradeMessage) From 82b8cbf847e76998cf61c6d51ef7ab94ba8b2742 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:15 +0100 Subject: [PATCH 063/144] docs: add Doxygen docs to auctions.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/analytics/auctions.hpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/include/itch/analytics/auctions.hpp b/include/itch/analytics/auctions.hpp index 5e55df3..58349b2 100644 --- a/include/itch/analytics/auctions.hpp +++ b/include/itch/analytics/auctions.hpp @@ -1,5 +1,14 @@ #pragma once +/// @file +/// @brief Reconstruction of Nasdaq auction (cross) events from the ITCH feed. +/// +/// Combines the Net Order Imbalance Indicator (NOII) messages leading up to a +/// cross with the Cross Trade message that prints it, producing a single +/// `Auction` record per opening, closing, halt, or IPO cross. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -36,6 +45,9 @@ struct Auction { }; /// @brief A human-readable description of a cross type code. +/// +/// @param cross_type The single-character cross type code (e.g. 'O', 'C', 'H', 'I'). +/// @return A short description of `cross_type`, or "Unknown" if unrecognized. [[nodiscard]] inline auto cross_type_name(char cross_type) -> std::string_view { constexpr indicators::FrozenMap types {std::to_array>({ {'O', "Opening Cross"}, @@ -57,11 +69,16 @@ class AuctionTracker { using AuctionCallback = std::function; /// @brief Installs the callback invoked for each reconstructed auction. + /// + /// @param callback Function invoked with each `Auction` reconstructed from the stream. auto set_auction_callback(AuctionCallback callback) -> void { m_callback = std::move(callback); } /// @brief Processes one ITCH message, emitting an auction on each cross. + /// + /// @param message The ITCH message to process; only NOII and Cross Trade messages + /// affect state. auto process(const Message& message) -> void { if (const auto* noii = std::get_if(&message)) { m_latest_imbalance[noii->stock_locate] = make_imbalance_info(*noii); From d9b831fbb9f006c878b848452bc3147b31a4f676 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:15 +0100 Subject: [PATCH 064/144] docs: add Doxygen docs to bars.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/analytics/bars.hpp | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/include/itch/analytics/bars.hpp b/include/itch/analytics/bars.hpp index e40efca..2ef941f 100644 --- a/include/itch/analytics/bars.hpp +++ b/include/itch/analytics/bars.hpp @@ -1,5 +1,14 @@ #pragma once +/// @file +/// @brief OHLCV bar aggregation from the trade tape under configurable clocks. +/// +/// `BarBuilder` consumes a stream of trades and emits completed bars whenever a +/// pluggable clock policy (time-, tick-, or volume-based) reports a new bucket, +/// letting time bars, tick bars, and volume bars share one implementation. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -29,6 +38,11 @@ using BarCallback = std::function; /// @brief A clock that buckets trades by wall-clock time (nanoseconds). struct TimeClock { std::uint64_t interval_ns {1}; + + /// @brief Computes the bucket id for `trade` from its wall-clock timestamp. + /// + /// @param trade The trade whose timestamp determines the bucket. + /// @return The bucket id, `trade.timestamp / interval_ns`. [[nodiscard]] auto bucket(const Trade& trade, std::uint64_t, std::uint64_t) const noexcept -> std::uint64_t { return trade.timestamp / interval_ns; @@ -38,6 +52,11 @@ struct TimeClock { /// @brief A clock that buckets a fixed number of trades into each bar. struct TickClock { std::uint64_t ticks_per_bar {1}; + + /// @brief Computes the bucket id for the current trade from its tick position. + /// + /// @param tick_index 0-based index of the trade within the stream. + /// @return The bucket id, `tick_index / ticks_per_bar`. [[nodiscard]] auto bucket(const Trade&, std::uint64_t, std::uint64_t tick_index) const noexcept -> std::uint64_t { return tick_index / ticks_per_bar; @@ -47,6 +66,11 @@ struct TickClock { /// @brief A clock that buckets a fixed amount of traded volume into each bar. struct VolumeClock { std::uint64_t volume_per_bar {1}; + + /// @brief Computes the bucket id for the current trade from cumulative volume. + /// + /// @param cumulative_volume Total shares traded so far, including this trade. + /// @return The bucket id, `cumulative_volume / volume_per_bar`. [[nodiscard]] auto bucket( const Trade&, std::uint64_t cumulative_volume, std::uint64_t ) const noexcept -> std::uint64_t { @@ -63,10 +87,17 @@ struct VolumeClock { template class BarBuilder { public: + /// @brief Constructs a builder with the given clock policy and completion callback. + /// + /// @param clock Bucketing policy (e.g. `TimeClock`, `TickClock`, `VolumeClock`) + /// that decides when a bar completes. + /// @param callback Function invoked with each completed `Bar`. BarBuilder(Clock clock, BarCallback callback) : m_clock {std::move(clock)}, m_callback {std::move(callback)} {} /// @brief Adds one trade, emitting the previous bar when the bucket changes. + /// + /// @param trade The next trade in timestamp order to fold into the current bar. auto add(const Trade& trade) -> void { const std::uint64_t bucket = m_clock.bucket(trade, m_cumulative_volume, m_tick_index); if (m_has_bar && bucket != m_bar.bucket) { @@ -100,6 +131,7 @@ class BarBuilder { } private: + /// @brief Delivers the current bar to the callback and marks it inactive. auto emit() -> void { if (m_callback) { m_callback(m_bar); From ca71a3dfd1e8c7684902426078e6f9fe866260b2 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:15 +0100 Subject: [PATCH 065/144] docs: add Doxygen docs to imbalance.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/analytics/imbalance.hpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/include/itch/analytics/imbalance.hpp b/include/itch/analytics/imbalance.hpp index 75261e7..707b75a 100644 --- a/include/itch/analytics/imbalance.hpp +++ b/include/itch/analytics/imbalance.hpp @@ -1,5 +1,14 @@ #pragma once +/// @file +/// @brief Decoded view of Net Order Imbalance Indicator (NOII) messages. +/// +/// Wraps the raw NOII (`I`) message fields in `ImbalanceInfo`, using strong price +/// types and a trimmed symbol, for use by auction reconstruction and other +/// imbalance-aware analytics. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -32,6 +41,9 @@ struct ImbalanceInfo { }; /// @brief Builds an `ImbalanceInfo` from a raw NOII message. +/// +/// @param msg The raw NOII message to decode. +/// @return An `ImbalanceInfo` populated from `msg`'s fields. [[nodiscard]] inline auto make_imbalance_info(const NOIIMessage& msg) -> ImbalanceInfo { ImbalanceInfo info {}; info.timestamp = msg.timestamp; @@ -49,6 +61,9 @@ struct ImbalanceInfo { } /// @brief A human-readable description of an imbalance direction code. +/// +/// @param direction The single-character imbalance direction code (e.g. 'B', 'S', 'N'). +/// @return A short description of `direction`, or "Unknown" if unrecognized. [[nodiscard]] inline auto imbalance_direction_name(char direction) -> std::string_view { constexpr indicators::FrozenMap directions {std::to_array>({ {'B', "Buy imbalance"}, From 2af4729ceab33a5b4968e5e1c998644fc6fc705f Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:16 +0100 Subject: [PATCH 066/144] docs: add Doxygen docs to microstructure.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/analytics/microstructure.hpp | 28 +++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/include/itch/analytics/microstructure.hpp b/include/itch/analytics/microstructure.hpp index 6650676..9c874e6 100644 --- a/include/itch/analytics/microstructure.hpp +++ b/include/itch/analytics/microstructure.hpp @@ -1,5 +1,13 @@ #pragma once +/// @file +/// @brief Best-bid-offer microstructure metrics derived from the order book. +/// +/// Small, allocation-free functions that compute spread, mid price, queue +/// imbalance, depth, and order-flow imbalance from `Bbo`/`L3Book` snapshots. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -9,6 +17,9 @@ namespace itch::analytics { /// @brief The bid-ask spread in price units, or NaN if either side is empty. +/// +/// @param bbo The best-bid-offer snapshot to compute the spread from. +/// @return `ask_price - bid_price`, or NaN if either side has no resting order. [[nodiscard]] inline auto spread(const book::Bbo& bbo) -> double { if (!bbo.has_bid || !bbo.has_ask) { return std::numeric_limits::quiet_NaN(); @@ -17,6 +28,10 @@ namespace itch::analytics { } /// @brief The mid price ((bid + ask) / 2), or NaN if either side is empty. +/// +/// @param bbo The best-bid-offer snapshot to compute the mid price from. +/// @return The average of the bid and ask prices, or NaN if either side has no +/// resting order. [[nodiscard]] inline auto mid_price(const book::Bbo& bbo) -> double { if (!bbo.has_bid || !bbo.has_ask) { return std::numeric_limits::quiet_NaN(); @@ -29,6 +44,10 @@ namespace itch::analytics { /// Positive values indicate more size resting on the bid than the offer /// (buy-side pressure); negative values the reverse. Returns NaN when there is no /// resting size at the top of either side. +/// +/// @param bbo The best-bid-offer snapshot to compute the queue imbalance from. +/// @return The normalized bid/ask size imbalance in [-1, 1], or NaN if both sides +/// are empty. [[nodiscard]] inline auto queue_imbalance(const book::Bbo& bbo) -> double { const double bid = static_cast(bbo.bid_shares); const double ask = static_cast(bbo.ask_shares); @@ -40,6 +59,11 @@ namespace itch::analytics { } /// @brief Total displayed shares within the best `levels` price levels of a side. +/// +/// @param book The order book to query. +/// @param side The book side (bid or ask) to sum depth for. +/// @param levels Number of best price levels to include. +/// @return The total displayed shares across the requested levels. [[nodiscard]] inline auto depth_at_level( const book::L3Book& book, book::Side side, std::size_t levels ) -> std::uint64_t { @@ -56,6 +80,10 @@ namespace itch::analytics { /// Stoikov: each side compares the new best price and size against the previous to /// attribute added or removed liquidity, and the offer contribution is subtracted /// from the bid contribution. Positive values indicate net buy-side flow. +/// +/// @param previous The prior best-bid-offer snapshot. +/// @param current The current best-bid-offer snapshot. +/// @return The signed order-flow-imbalance contribution between the two snapshots. [[nodiscard]] inline auto order_flow_imbalance(const book::Bbo& previous, const book::Bbo& current) -> double { const double prev_bid_price = previous.bid_price.to_double(); From e993d236ec1d3e811bce102296683be3de0e8f48 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:16 +0100 Subject: [PATCH 067/144] docs: add Doxygen docs to vwap.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/analytics/vwap.hpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/include/itch/analytics/vwap.hpp b/include/itch/analytics/vwap.hpp index 668de29..607cf97 100644 --- a/include/itch/analytics/vwap.hpp +++ b/include/itch/analytics/vwap.hpp @@ -1,5 +1,14 @@ #pragma once +/// @file +/// @brief Running VWAP and TWAP accumulators for streaming price and volume data. +/// +/// These lightweight accumulators are fed one execution or price sample at a time +/// and report a running volume-weighted or time-weighted average price without +/// retaining the full history of samples. +/// +/// @author Bertin Balouki SIMYELI + #include #include @@ -14,12 +23,17 @@ namespace itch::analytics { class Vwap { public: /// @brief Adds an execution of `shares` at `price` to the accumulation. + /// + /// @param price Execution price of the trade being added. + /// @param shares Number of shares traded at `price`. auto add(StandardPrice price, std::uint64_t shares) -> void { m_price_volume += price.to_double() * static_cast(shares); m_volume += shares; } /// @brief The current VWAP, or NaN when no volume has been added. + /// + /// @return The volume-weighted average price, or NaN if `volume()` is zero. [[nodiscard]] auto value() const -> double { if (m_volume == 0) { return std::numeric_limits::quiet_NaN(); @@ -28,6 +42,8 @@ class Vwap { } /// @brief The total shares accumulated so far. + /// + /// @return The sum of shares passed to `add` since construction or the last `reset`. [[nodiscard]] auto volume() const noexcept -> std::uint64_t { return m_volume; } /// @brief Clears the accumulation (start a new interval). @@ -49,6 +65,9 @@ class Vwap { class Twap { public: /// @brief Records that the price became `price` at `timestamp` (ns). + /// + /// @param price The prevailing price starting at `timestamp`. + /// @param timestamp Time of the price change, in nanoseconds. auto add(StandardPrice price, std::uint64_t timestamp) -> void { if (m_has_sample && timestamp > m_last_timestamp) { const double elapsed = static_cast(timestamp - m_last_timestamp); @@ -61,6 +80,8 @@ class Twap { } /// @brief The current TWAP, or NaN when no span has elapsed. + /// + /// @return The time-weighted average price, or NaN if no sample has been added. [[nodiscard]] auto value() const -> double { if (m_total_time <= 0.0) { return m_has_sample ? m_last_price : std::numeric_limits::quiet_NaN(); From a0ac8b7e9043d1894c2fb86465b0021664557403 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:16 +0100 Subject: [PATCH 068/144] docs: add Doxygen docs to book_manager.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/book/book_manager.hpp | 50 +++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/include/itch/book/book_manager.hpp b/include/itch/book/book_manager.hpp index 329ae6c..5a531dc 100644 --- a/include/itch/book/book_manager.hpp +++ b/include/itch/book/book_manager.hpp @@ -1,5 +1,16 @@ #pragma once +/// @file +/// @brief Full-market order book manager that fans out ITCH messages to +/// per-symbol `L3Book` instances. +/// +/// This header declares `BookManager`, which owns a locate-indexed table of +/// `L3Book`s, routes each parsed ITCH message to the right book, optionally +/// restricts work to a configured symbol universe, and emits best-bid/offer +/// and trade events as they occur. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -28,34 +39,50 @@ class BookManager { /// @brief Invoked when a book's best bid or offer changes. using BboCallback = std::function; + /// @brief Constructs an empty manager with no books tracked yet. BookManager() = default; /// @brief Processes one parsed ITCH message, updating the relevant book and /// emitting BBO/trade events as appropriate. + /// @param message The parsed ITCH message to apply. auto process(const Message& message) -> void; /// @brief Installs the best-bid/offer change callback (empty clears it). + /// @param callback Invoked whenever a tracked book's best bid or offer + /// changes. auto set_bbo_callback(BboCallback callback) -> void { m_bbo_callback = std::move(callback); } /// @brief Installs the trade-tape callback (empty clears it). + /// @param callback Invoked for each trade extracted from the feed. auto set_trade_callback(TradeCallback callback) -> void { m_trade_callback = std::move(callback); } /// @brief Restricts tracking to the given symbol (call once per symbol). When /// no symbol is added, every symbol on the feed is tracked. + /// @param symbol The ticker symbol to add to the tracked universe. auto track_symbol(std::string symbol) -> void { m_universe.insert(std::move(symbol)); } /// @brief The book for a locate code, or nullptr if none is tracked. + /// @param stock_locate The exchange-assigned stock locate code. + /// @return Pointer to the book for `stock_locate`, or nullptr if it is + /// not tracked. [[nodiscard]] auto book(std::uint16_t stock_locate) const -> const L3Book*; /// @brief The book for a symbol, or nullptr if none is tracked. + /// @param symbol The ticker symbol to look up. + /// @return Pointer to the book for `symbol`, or nullptr if it is not + /// tracked. [[nodiscard]] auto book_for_symbol(std::string_view symbol) const -> const L3Book*; /// @brief The number of books currently maintained. + /// @return The count of books currently tracked. [[nodiscard]] auto book_count() const noexcept -> std::size_t { return m_book_count; } /// @brief The symbol associated with a locate code (empty if unknown). + /// @param stock_locate The exchange-assigned stock locate code. + /// @return The ticker symbol for `stock_locate`, or an empty view if + /// unknown. [[nodiscard]] auto symbol_for_locate(std::uint16_t stock_locate) const -> std::string_view; private: @@ -64,13 +91,28 @@ class BookManager { Bbo last_bbo {}; }; - // Returns the entry for a locate, creating it if the symbol is in-universe. + /// @brief Returns the entry for a locate, creating it if the symbol is + /// in-universe. + /// @param stock_locate The exchange-assigned stock locate code. + /// @param symbol The ticker symbol associated with `stock_locate`. + /// @return Pointer to the (possibly newly created) entry, or nullptr if + /// `symbol` is not in the tracked universe. auto ensure_entry(std::uint16_t stock_locate, std::string_view symbol) -> BookEntry*; - // Returns the existing entry for a locate, or nullptr. + + /// @brief Returns the existing entry for a locate, or nullptr. + /// @param stock_locate The exchange-assigned stock locate code. + /// @return Pointer to the entry for `stock_locate`, or nullptr if none + /// exists. [[nodiscard]] auto entry(std::uint16_t stock_locate) const -> BookEntry*; - // Emits a BBO event if the book's top has changed since last seen. + + /// @brief Emits a BBO event if the book's top has changed since last seen. + /// @param target The book entry to check and, if changed, report. auto emit_bbo_if_changed(BookEntry& target) -> void; - // Whether the symbol should be tracked given the configured universe. + + /// @brief Whether the symbol should be tracked given the configured + /// universe. + /// @param symbol The ticker symbol to check. + /// @return True if `symbol` should be tracked, false otherwise. [[nodiscard]] auto in_universe(std::string_view symbol) const -> bool; std::vector> m_books_by_locate; ///< Indexed by locate. From ca9637a5030a65d37edb052d4634fca1be100109 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:17 +0100 Subject: [PATCH 069/144] docs: add Doxygen docs to l3_book.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/book/l3_book.hpp | 95 +++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/include/itch/book/l3_book.hpp b/include/itch/book/l3_book.hpp index 2235db2..8e9776e 100644 --- a/include/itch/book/l3_book.hpp +++ b/include/itch/book/l3_book.hpp @@ -1,5 +1,17 @@ #pragma once +/// @file +/// @brief Single-symbol, order-level (L3) limit order book with +/// allocation-light internals. +/// +/// This header declares `L3Book`, which reconstructs one security's full +/// order book from ITCH add/execute/cancel/delete/replace messages using an +/// object pool and intrusive FIFO queues instead of per-order heap +/// allocation, plus the small `Side`, `DepthLevel`, `OrderView`, and `Bbo` +/// value types used to query it. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -39,6 +51,11 @@ struct Bbo { StandardPrice ask_price {}; ///< Best (lowest) ask price. std::uint64_t ask_shares {0}; ///< Shares at the best ask. + /// @brief Compares two BBO snapshots for equality of all fields. + /// @param lhs The first snapshot to compare. + /// @param rhs The second snapshot to compare. + /// @return True if `lhs` and `rhs` have equal has_bid/has_ask/prices/ + /// shares, false otherwise. [[nodiscard]] friend auto operator==(const Bbo&, const Bbo&) noexcept -> bool = default; }; @@ -55,32 +72,59 @@ struct Bbo { class L3Book { public: /// @brief Constructs a book, optionally tagged with its stock symbol. + /// @param symbol The ticker symbol to associate with the book (may be + /// empty). explicit L3Book(std::string symbol = {}); /// @brief Sets the stock symbol associated with this book. + /// @param symbol The ticker symbol to associate with the book. auto set_symbol(std::string symbol) -> void { m_symbol = std::move(symbol); } /// @brief The stock symbol associated with this book (may be empty). + /// @return Const reference to the associated ticker symbol. [[nodiscard]] auto symbol() const noexcept -> const std::string& { return m_symbol; } /// @brief Adds a new resting order to the book (ITCH `A`/`F`). + /// @param reference_number Exchange order reference number identifying + /// the new order. + /// @param side The side of the book the order rests on. + /// @param shares The number of shares in the new order. + /// @param price The raw (unscaled) limit price of the new order. auto add_order( std::uint64_t reference_number, Side side, std::uint32_t shares, std::uint32_t price ) -> void; /// @brief Removes `shares` from an order on execution (ITCH `E`/`C`), /// deleting it when fully filled. Returns the shares actually removed. + /// @param reference_number Exchange order reference number of the order + /// being executed against. + /// @param shares The number of shares to remove from the order. + /// @return The number of shares actually removed (may be less than + /// `shares` if the order does not have that many resting). auto execute_order(std::uint64_t reference_number, std::uint32_t shares) -> std::uint32_t; /// @brief Removes `shares` from an order on a partial cancel (ITCH `X`), /// deleting it when it reaches zero. Returns the shares actually removed. + /// @param reference_number Exchange order reference number of the order + /// being reduced. + /// @param shares The number of shares to remove from the order. + /// @return The number of shares actually removed (may be less than + /// `shares` if the order does not have that many resting). auto reduce_order(std::uint64_t reference_number, std::uint32_t shares) -> std::uint32_t; /// @brief Deletes an order in its entirety (ITCH `D`). + /// @param reference_number Exchange order reference number of the order + /// to delete. auto delete_order(std::uint64_t reference_number) -> void; /// @brief Replaces an order with a new reference number, size, and price /// on the same side (ITCH `U`). + /// @param old_reference_number Exchange order reference number of the + /// order being replaced. + /// @param new_reference_number Exchange order reference number assigned + /// to the replacement order. + /// @param shares The number of shares in the replacement order. + /// @param price The raw (unscaled) limit price of the replacement order. auto replace_order( std::uint64_t old_reference_number, std::uint64_t new_reference_number, @@ -89,32 +133,49 @@ class L3Book { ) -> void; /// @brief Whether an order with the given reference number is resting. + /// @param reference_number Exchange order reference number to look up. + /// @return True if an order with `reference_number` is resting, false + /// otherwise. [[nodiscard]] auto contains(std::uint64_t reference_number) const -> bool; /// @brief The raw limit price of a resting order, if present. + /// @param reference_number Exchange order reference number to look up. + /// @return The order's raw limit price, or `std::nullopt` if no such + /// order is resting. [[nodiscard]] auto order_price(std::uint64_t reference_number) const -> std::optional; /// @brief The side of a resting order, if present. + /// @param reference_number Exchange order reference number to look up. + /// @return The order's side, or `std::nullopt` if no such order is + /// resting. [[nodiscard]] auto order_side(std::uint64_t reference_number) const -> std::optional; /// @brief The current best bid and offer. + /// @return The book's current best bid and offer snapshot. [[nodiscard]] auto bbo() const -> Bbo; /// @brief Aggregated L2 depth for a side, best level first. /// /// @param side The side to report. /// @param max_levels The maximum number of levels to return (0 == all). + /// @return The requested side's price levels, best (top of book) first. [[nodiscard]] auto depth(Side side, std::size_t max_levels = 0) const -> std::vector; /// @brief The resting orders at a given price on a side, in time priority. + /// @param side The side to query. + /// @param price The raw (unscaled) limit price to query. + /// @return The resting orders at `price` on `side`, oldest first. [[nodiscard]] auto orders_at(Side side, std::uint32_t price) const -> std::vector; /// @brief The number of active price levels on a side. + /// @param side The side to query. + /// @return The count of active price levels on `side`. [[nodiscard]] auto level_count(Side side) const noexcept -> std::size_t; /// @brief Whether the book has no resting orders on either side. + /// @return True if no orders are resting on either side, false otherwise. [[nodiscard]] auto empty() const noexcept -> bool { return m_index.empty(); } private: @@ -140,18 +201,42 @@ class L3Book { std::uint32_t tail {NIL}; }; + /// @brief The mutable level ladder for a side. + /// @param side The side whose ladder to return. + /// @return Reference to the vector of price levels for `side`. [[nodiscard]] auto side_levels(Side side) noexcept -> std::vector&; + + /// @brief The level ladder for a side. + /// @param side The side whose ladder to return. + /// @return Const reference to the vector of price levels for `side`. [[nodiscard]] auto side_levels(Side side) const noexcept -> const std::vector&; - // Allocates a node from the free list (or grows the pool) and returns its index. + /// @brief Allocates a node from the free list (or grows the pool) and + /// returns its index. + /// @return The pool index of the newly allocated node. auto allocate_node() -> std::uint32_t; - // Returns a node to the free list. + + /// @brief Returns a node to the free list. + /// @param node_index The pool index of the node to free. auto free_node(std::uint32_t node_index) -> void; - // Finds the index of the level at `price` on `side`, or NIL if absent. + + /// @brief Finds the index of the level at `price` on `side`, or NIL if + /// absent. + /// @param side The side to search. + /// @param price The raw (unscaled) limit price to find. + /// @return The index of the matching level, or `NIL` if none exists. [[nodiscard]] auto find_level(Side side, std::uint32_t price) const -> std::uint32_t; - // Finds, or inserts in sorted order, the level at `price`; returns its index. + + /// @brief Finds, or inserts in sorted order, the level at `price`; + /// returns its index. + /// @param side The side to search or insert into. + /// @param price The raw (unscaled) limit price to find or create. + /// @return The index of the existing or newly created level. auto find_or_create_level(Side side, std::uint32_t price) -> std::uint32_t; - // Unlinks a node from its level FIFO and removes the level if it empties. + + /// @brief Unlinks a node from its level FIFO and removes the level if it + /// empties. + /// @param node_index The pool index of the node to unlink. auto unlink_node(std::uint32_t node_index) -> void; std::string m_symbol; From 4a430912e4be5724545d5181fb296d611effcdda Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:17 +0100 Subject: [PATCH 070/144] docs: add Doxygen docs to order_index.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/book/order_index.hpp | 36 +++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/include/itch/book/order_index.hpp b/include/itch/book/order_index.hpp index 19b49a3..fc73e18 100644 --- a/include/itch/book/order_index.hpp +++ b/include/itch/book/order_index.hpp @@ -1,5 +1,16 @@ #pragma once +/// @file +/// @brief Allocation-light, open-addressed hash map from order reference +/// number to order-pool index. +/// +/// This header declares `OrderIndex`, the lookup structure `L3Book` uses to +/// resolve an ITCH order reference number to its slot in the order pool in +/// O(1) without the per-insert/per-erase heap allocations of +/// `std::unordered_map`. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -20,15 +31,21 @@ class OrderIndex { /// @brief Sentinel returned by `find` when a key is absent. static constexpr std::uint32_t NPOS = 0xFFFFFFFFU; + /// @brief Constructs an empty map with the initial table capacity + /// pre-allocated. OrderIndex() { rehash(INITIAL_CAPACITY); } /// @brief The number of stored keys. + /// @return The count of stored keys. [[nodiscard]] auto size() const noexcept -> std::size_t { return m_count; } /// @brief Whether the map holds no keys. + /// @return True if the map holds no keys, false otherwise. [[nodiscard]] auto empty() const noexcept -> bool { return m_count == 0; } /// @brief Returns the value for `key`, or `NPOS` if absent. + /// @param key The order reference number to look up. + /// @return The stored pool index for `key`, or `NPOS` if absent. [[nodiscard]] auto find(std::uint64_t key) const noexcept -> std::uint32_t { std::size_t slot = hash(key) & m_mask; while (m_slots[slot].used) { @@ -41,11 +58,15 @@ class OrderIndex { } /// @brief Whether `key` is present. + /// @param key The order reference number to check. + /// @return True if `key` is present, false otherwise. [[nodiscard]] auto contains(std::uint64_t key) const noexcept -> bool { return find(key) != NPOS; } /// @brief Inserts or overwrites the value for `key`. + /// @param key The order reference number to insert or update. + /// @param value The pool index to associate with `key`. auto insert(std::uint64_t key, std::uint32_t value) -> void { if ((m_count + 1) * LOAD_FACTOR_DEN >= m_slots.size() * LOAD_FACTOR_NUM) { rehash(m_slots.size() * 2); @@ -63,6 +84,7 @@ class OrderIndex { } /// @brief Removes `key` if present, repairing the probe chain in place. + /// @param key The order reference number to remove. auto erase(std::uint64_t key) -> void { std::size_t slot = hash(key) & m_mask; while (m_slots[slot].used) { @@ -93,8 +115,11 @@ class OrderIndex { static constexpr std::size_t LOAD_FACTOR_NUM = 7; // Grow past 70% load. static constexpr std::size_t LOAD_FACTOR_DEN = 10; - // A finalizing mix (splitmix64) so dense, sequential reference numbers spread - // across the table instead of clustering. + /// @brief Computes a finalizing hash mix (splitmix64) of a key so dense, + /// sequential reference numbers spread across the table instead of + /// clustering. + /// @param key The order reference number to hash. + /// @return The mixed hash value used to select a probe slot. [[nodiscard]] static auto hash(std::uint64_t key) noexcept -> std::size_t { key ^= key >> 33; key *= 0xFF51AFD7ED558CCDULL; @@ -104,6 +129,10 @@ class OrderIndex { return static_cast(key); } + /// @brief Removes the entry at `hole` and backward-shifts any entries in + /// its probe chain so lookups for other keys are not broken by the + /// gap. + /// @param hole The slot index of the entry to remove. auto remove_at(std::size_t hole) -> void { m_slots[hole].used = false; --m_count; @@ -124,6 +153,9 @@ class OrderIndex { } } + /// @brief Grows (or initializes) the table to `new_capacity` and + /// re-inserts every previously stored key. + /// @param new_capacity The new table capacity, must be a power of two. auto rehash(std::size_t new_capacity) -> void { std::vector old = std::move(m_slots); m_slots.assign(new_capacity, Slot {}); From 26cb49690cb735ba4cde1b516f365b82971bcdc1 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:17 +0100 Subject: [PATCH 071/144] docs: add Doxygen docs to wire.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/detail/wire.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/itch/detail/wire.hpp b/include/itch/detail/wire.hpp index 93589c1..33cc4c6 100644 --- a/include/itch/detail/wire.hpp +++ b/include/itch/detail/wire.hpp @@ -12,6 +12,8 @@ /// Keeping the per-type wire size and the canonical list of message types in one /// place means the dispatch table, the overlay accessors, and the writer can /// never drift apart from one another. +/// +/// @author Bertin Balouki SIMYELI namespace itch::detail { @@ -39,6 +41,9 @@ static_assert(WIRE_SIZE == 48); /// This is the canonical registry of message types. Both the parser's dispatch /// table and any other component that needs to enumerate the message set build /// on it, so the set is defined exactly once. +/// +/// @tparam Visitor Callable type providing a templated `operator()(char)`. +/// @param visitor The visitor invoked once per registered message type. template constexpr auto for_each_message_type(Visitor&& visitor) -> void { visitor.template operator()('S'); From 25c6dd9118fdfd8774cb36ee5738dc0aef388b65 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:18 +0100 Subject: [PATCH 072/144] docs: add Doxygen docs to encoder.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/encoder.hpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/include/itch/encoder.hpp b/include/itch/encoder.hpp index 5ff5d9b..c01605c 100644 --- a/include/itch/encoder.hpp +++ b/include/itch/encoder.hpp @@ -1,12 +1,6 @@ #pragma once -#include -#include - -#include "itch/messages.hpp" - -namespace itch { - +/// @file /// @brief Serializes parsed ITCH messages back to valid wire bytes. /// /// The encoder is the inverse of the parser: it writes each message's fields in @@ -14,15 +8,28 @@ namespace itch { /// timestamp. It is used to synthesize valid ITCH streams for testing, scenario /// generation, and golden fixtures, and it guarantees the round-trip property /// `parse(encode(msg)) == msg` for every supported message type. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include + +#include "itch/messages.hpp" + +namespace itch { /// @brief Encodes a message body and header without the 2-byte length prefix. /// +/// @param message The parsed message to encode. /// @return The raw message bytes (type byte first), exactly as they appear inside /// a frame. [[nodiscard]] auto encode_message(const Message& message) -> std::vector; /// @brief Encodes a complete length-prefixed frame (2-byte big-endian length /// followed by the message bytes), ready to concatenate into a stream. +/// +/// @param message The parsed message to encode. +/// @return The length-prefixed frame bytes, ready to append to a stream. [[nodiscard]] auto encode_frame(const Message& message) -> std::vector; } // namespace itch From 4e35c66f42d1e62f5239e9c8c0280453e23ed662 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:18 +0100 Subject: [PATCH 073/144] docs: add Doxygen docs to indicators.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/indicators.hpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/include/itch/indicators.hpp b/include/itch/indicators.hpp index c2635c2..4060db7 100644 --- a/include/itch/indicators.hpp +++ b/include/itch/indicators.hpp @@ -1,5 +1,16 @@ #pragma once +/// @file +/// @brief Compile-time lookup tables translating ITCH single- and +/// multi-character indicator codes into human-readable descriptions. +/// +/// These tables back the enum-like character fields scattered across the ITCH +/// message definitions (system events, market categories, financial status, +/// trading actions, and so on), giving callers a fast, allocation-free way to +/// render a wire code as descriptive text. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -25,12 +36,20 @@ class FrozenMap { public: using value_type = std::pair; + /// @brief Constructs the map from an unsorted list of key/description + /// entries, sorting them by key once so that `find` can binary + /// search. + /// + /// @param entries The key/description pairs backing this map, in any order. constexpr explicit FrozenMap(std::array entries) noexcept : m_entries {entries} { std::ranges::sort(m_entries, {}, &value_type::first); } /// @brief Returns the description for a key, or `std::nullopt` if absent. + /// + /// @param key The key to look up. + /// @return The matching description, or `std::nullopt` if no entry has this key. [[nodiscard]] constexpr auto find(const KeyType& key) const noexcept -> std::optional { const auto iter = std::ranges::lower_bound(m_entries, key, {}, &value_type::first); @@ -41,6 +60,10 @@ class FrozenMap { } /// @brief Returns the description for a key, or a fallback if absent. + /// + /// @param key The key to look up. + /// @param fallback The description to return when `key` is not found. + /// @return The matching description, or `fallback` if no entry has this key. [[nodiscard]] constexpr auto at_or(const KeyType& key, std::string_view fallback) const noexcept -> std::string_view { return find(key).value_or(fallback); @@ -50,6 +73,12 @@ class FrozenMap { std::array m_entries; }; +/// @brief Deduces `FrozenMap`'s template arguments from a brace-initialized +/// array of key/description pairs. +/// +/// @tparam KeyType The key type (`char` for single-letter codes, or +/// `std::string_view` for multi-character codes). +/// @tparam Size The number of entries. template FrozenMap(std::array, Size>) -> FrozenMap; From 6c9182c099ef6372ffa805d7fdc02e59e0fd70eb Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:18 +0100 Subject: [PATCH 074/144] docs: add Doxygen docs to arrow_export.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/io/arrow_export.hpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/include/itch/io/arrow_export.hpp b/include/itch/io/arrow_export.hpp index 363fdcc..78cc5f8 100644 --- a/include/itch/io/arrow_export.hpp +++ b/include/itch/io/arrow_export.hpp @@ -8,6 +8,8 @@ /// is off the class is not compiled, keeping the core dependency-free. The export /// turns a feed into a columnar table that drops straight into pandas, Polars, /// DuckDB, and Spark pipelines. +/// +/// @author Bertin Balouki SIMYELI #ifdef ITCH_WITH_ARROW @@ -31,25 +33,48 @@ namespace itch::io { /// Append each message with `append`, then call `write_parquet` to flush a file. class ArrowExporter { public: + /// @brief Constructs an empty exporter with no rows appended. ArrowExporter(); + + /// @brief Destroys the exporter, releasing any held Arrow/Parquet resources. ~ArrowExporter(); + + /// @brief Non-copyable; move-only. ArrowExporter(const ArrowExporter&) = delete; + + /// @brief Non-copyable; move-only. + /// + /// @return Reference to this exporter. auto operator=(const ArrowExporter&) -> ArrowExporter& = delete; + + /// @brief Move-constructs an exporter, transferring ownership of its state. ArrowExporter(ArrowExporter&&) noexcept = default; + + /// @brief Move-assigns an exporter, transferring ownership of its state. + /// + /// @return Reference to this exporter. auto operator=(ArrowExporter&&) noexcept -> ArrowExporter& = default; /// @brief Appends one message as a row to the column builders. + /// + /// @param message The parsed message to append. auto append(const Message& message) -> void; /// @brief The number of rows appended so far. + /// + /// @return The count of rows appended since construction. [[nodiscard]] auto rows() const noexcept -> std::uint64_t; /// @brief Finishes the builders and writes a Parquet file to `path`. /// + /// @param path Filesystem path to write the Parquet file to. /// @return True on success; on failure `error()` holds a description. [[nodiscard]] auto write_parquet(const std::string& path) -> bool; /// @brief A description of the last failure, or empty on success. + /// + /// @return The last failure message, or an empty string if the last operation + /// succeeded. [[nodiscard]] auto error() const -> const std::string&; private: From 3d665310f569dda31c102e18053cbfab056a432d Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:19 +0100 Subject: [PATCH 075/144] docs: add Doxygen docs to csv_sink.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/io/csv_sink.hpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/itch/io/csv_sink.hpp b/include/itch/io/csv_sink.hpp index 09faca2..a5dd90b 100644 --- a/include/itch/io/csv_sink.hpp +++ b/include/itch/io/csv_sink.hpp @@ -1,5 +1,14 @@ #pragma once +/// @file +/// @brief CSV output sink for parsed ITCH messages. +/// +/// `CsvSink` is the dependency-free, universal output format: it flattens the +/// heterogeneous ITCH message set into one wide, normalized row schema suitable +/// for quick inspection and legacy tooling. +/// +/// @author Bertin Balouki SIMYELI + #include #include @@ -22,15 +31,22 @@ class CsvSink : public MessageSink { public: /// @brief Constructs a sink writing to `out`, emitting the header row unless /// `write_header` is false (useful when appending to an existing file). + /// + /// @param out Output stream the CSV rows are written to. + /// @param write_header Whether to emit the CSV header row on construction. explicit CsvSink(std::ostream& out, bool write_header = true); /// @brief Writes one message as a CSV row. + /// + /// @param message The parsed message to write. auto write(const Message& message) -> void override; /// @brief Flushes the underlying stream. auto flush() -> void override; /// @brief The number of rows written so far. + /// + /// @return The count of rows written since construction. [[nodiscard]] auto rows_written() const noexcept -> std::uint64_t { return m_rows_written; } private: From 64df62eaa87e8ece6a60a0612d6949d3c1325392 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:19 +0100 Subject: [PATCH 076/144] docs: add Doxygen docs to sink.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/io/sink.hpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/include/itch/io/sink.hpp b/include/itch/io/sink.hpp index 5390ffd..7656df0 100644 --- a/include/itch/io/sink.hpp +++ b/include/itch/io/sink.hpp @@ -1,5 +1,14 @@ #pragma once +/// @file +/// @brief Abstract streaming sink interface for parsed ITCH messages. +/// +/// `MessageSink` is the common output interface implemented by `CsvSink`, +/// `ArrowExporter`, and other consumers that receive one message at a time from +/// a parse callback. +/// +/// @author Bertin Balouki SIMYELI + #include "itch/messages.hpp" namespace itch::io { @@ -11,14 +20,31 @@ namespace itch::io { /// counter). Implementations override `write`; `flush` is optional. class MessageSink { public: + /// @brief Default-constructs an empty sink. MessageSink() = default; + + /// @brief Copy-constructs a sink. MessageSink(const MessageSink&) = default; + + /// @brief Move-constructs a sink. MessageSink(MessageSink&&) noexcept = default; + + /// @brief Copy-assigns a sink. + /// + /// @return Reference to this sink. auto operator=(const MessageSink&) -> MessageSink& = default; + + /// @brief Move-assigns a sink. + /// + /// @return Reference to this sink. auto operator=(MessageSink&&) noexcept -> MessageSink& = default; + + /// @brief Virtual destructor for safe polymorphic destruction. virtual ~MessageSink() = default; /// @brief Consumes one parsed message. + /// + /// @param message The parsed message to write. virtual auto write(const Message& message) -> void = 0; /// @brief Flushes any buffered output. The default is a no-op. From d5570aeb3d84b4d7bef3cd9f31f6d4812094556f Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:19 +0100 Subject: [PATCH 077/144] docs: add Doxygen docs to messages.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/messages.hpp | 66 ++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/include/itch/messages.hpp b/include/itch/messages.hpp index 56abe64..1f8a24a 100644 --- a/include/itch/messages.hpp +++ b/include/itch/messages.hpp @@ -1,5 +1,18 @@ #pragma once +/// @file +/// @brief Defines every ITCH 5.0 message struct, the `Message` variant, and the +/// stream-printing helpers used to format them. +/// +/// The TotalView ITCH feed is composed of a series of messages that describe +/// orders added to, removed from, and executed on Nasdaq, as well as Cross and +/// Stock Directory information. Each message begins with a one-byte Message Type +/// field that identifies the structure of the remainder of the message; this +/// header defines one struct per message type, the `Message` variant able to hold +/// any of them, and free functions to print a message to a stream. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -425,21 +438,14 @@ struct DLCRMessage { // RESTORE DEFAULT PADDING #pragma pack(pop) -/// The TotalView ITCH feed is composed of a series of messages that describe -/// orders added to, removed from, and executed on Nasdaq as well as disseminate -/// Cross and Stock Directory information. -/// Each message begins with a one-byte Message Type field that identifies the -/// structure of the remainder of the message. The Message Type is followed by -/// fields that are specific to each message type. -/// -/// All Message have the following attributes: -/// - message_type: A single letter that identify the message -/// - timestamp: Time at which the message was generated (Nanoseconds past -/// midnight) -/// - stock_locate: Locate code identifying the security -/// - tracking_number: Nasdaq internal tracking number -/// -/// for more details on each message type, see the +/// @brief A variant able to hold any one of the ITCH 5.0 message structs. +/// +/// The Message Type byte at the start of each frame identifies which struct is +/// active. All alternatives share four common attributes: `message_type` (a +/// single letter identifying the message), `timestamp` (nanoseconds past +/// midnight), `stock_locate` (locate code identifying the security), and +/// `tracking_number` (Nasdaq internal tracking number). For more details on each +/// message type, see the /// [documentation](https://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NQTVITCHspecification.pdf). /// /// @note @@ -479,7 +485,12 @@ constexpr int STOCK_LEN = 8; constexpr double PRICE_DIVISOR = 10000.0; constexpr double MWCB_PRICE_DIVISOR = 1.0E8; -// Convert char arrays to strings, trimming trailing spaces. +/// @brief Converts a fixed-width character array to a string, trimming trailing +/// spaces and NUL characters. +/// +/// @param source Pointer to the first character of the fixed-width field. +/// @param size Number of characters in the field. +/// @return The trimmed string, with trailing spaces and NUL characters removed. inline auto to_string(const char* source, size_t size) -> std::string { size_t len = size; while (len > 0 && (source[len - 1] == ' ' || source[len - 1] == '\0')) { @@ -488,6 +499,16 @@ inline auto to_string(const char* source, size_t size) -> std::string { return std::string {source, len}; } +/// @brief Per-message-type stream formatter family, used internally by +/// `print_message` to dispatch to the correct overload for whichever +/// alternative of `Message` it is given. +/// +/// One overload of `print_impl` exists for every message struct defined above. +/// Each overload writes a human-readable, field-by-field representation of its +/// message to `out`. +/// +/// @param out The output stream to write to. +/// @param msg The message to format. auto print_impl(std::ostream& out, const SystemEventMessage& msg) -> void; auto print_impl(std::ostream& out, const StockDirectoryMessage& msg) -> void; auto print_impl(std::ostream& out, const StockTradingActionMessage& msg) -> void; @@ -512,10 +533,19 @@ auto print_impl(std::ostream& out, const NOIIMessage& msg) -> void; auto print_impl(std::ostream& out, const RetailPriceImprovementIndicatorMessage& msg) -> void; auto print_impl(std::ostream& out, const DLCRMessage& msg) -> void; -// General print function that dispatches to the correct implementation +/// @brief Dispatches to the `print_impl` overload matching the active +/// alternative of `msg` and writes its formatted representation to `out`. +/// +/// @param out The output stream to write to. +/// @param msg The message to format; the concrete type formatted is whichever +/// alternative of the variant is currently held. auto print_message(std::ostream& out, const Message& msg) -> void; -// print an itch::Message +/// @brief Streams a formatted representation of `msg` to `out`. +/// +/// @param out The output stream to write to. +/// @param msg The message to format. +/// @return A reference to `out`, to allow chaining. auto operator<<(std::ostream& out, const Message& msg) -> std::ostream&; } // namespace itch From b9e4a41b5dc01f253d182344829ceff113b4267e Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:20 +0100 Subject: [PATCH 078/144] docs: add Doxygen docs to order_book.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/order_book.hpp | 72 ++++++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/include/itch/order_book.hpp b/include/itch/order_book.hpp index d0247d4..0f6be61 100644 --- a/include/itch/order_book.hpp +++ b/include/itch/order_book.hpp @@ -1,5 +1,16 @@ #pragma once +/// @file +/// @brief An in-memory limit order book reconstructed from a stream of parsed +/// ITCH 5.0 messages. +/// +/// `LimitOrderBook` maintains the resting bid/ask price levels for a single +/// instrument, applying Add/Execute/Cancel/Delete/Replace order events as they +/// arrive so callers can query the current book depth or render it for +/// inspection. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -29,6 +40,13 @@ struct Order { PriceLevel* level; ///< Pointer to the price level containing this order. std::string stock; ///< The stock symbol for this order. + /// @brief Constructs an order with the given identity, side, quantity, and price. + /// + /// @param ref_num The exchange-assigned order reference number. + /// @param side 'B' for Buy, 'S' for Sell. + /// @param shrs The initial quantity of shares in the order. + /// @param prc The limit price of the order. + /// @param stk The stock symbol for this order. Order(uint64_t ref_num, char side, uint32_t shrs, uint32_t prc, const std::string& stk) : order_reference_number {ref_num}, buy_sell_indicator {side}, @@ -71,6 +89,10 @@ struct PriceLevel { /// of price levels, and O(1) access to the best bid/ask. class LimitOrderBook { public: + /// @brief Constructs an order book scoped to a single stock symbol. + /// + /// @param stock_symbol The symbol of the instrument this book tracks; only + /// messages for this symbol affect the book state. LimitOrderBook(const std::string& stock_symbol) : m_stock_symbol(stock_symbol) {} /// @brief Dispatches and processes a generic ITCH message. /// @@ -94,9 +116,13 @@ class LimitOrderBook { using BidMap = std::map>; using AskMap = std::map>; + /// @brief The map of active Bids. + /// /// @return A reference to the map of active Bids, sorted descending by price. auto get_bids() const -> const BidMap& { return m_bids; } + /// @brief The map of active Asks. + /// /// @return A reference to the map of active Asks, sorted ascending by price. auto get_asks() const -> const AskMap& { return m_asks; } @@ -110,19 +136,63 @@ class LimitOrderBook { std::map m_orders; ///< Hash map for O(1) order lookup by Reference Number. // Message Handlers + /// @brief Handles an Add Order (no MPID) message: inserts a new resting + /// order into the book if it belongs to this instrument. + /// + /// @param msg The parsed Add Order message. auto handle_message(const AddOrderMessage& msg) -> void; + + /// @brief Handles an Add Order with MPID Attribution message: inserts a + /// new resting order into the book if it belongs to this instrument. + /// + /// @param msg The parsed Add Order (MPID Attribution) message. auto handle_message(const AddOrderMPIDAttributionMessage& msg) -> void; + + /// @brief Handles an Order Executed message: reduces or removes the + /// referenced order by its executed share quantity. + /// + /// @param msg The parsed Order Executed message. auto handle_message(const OrderExecutedMessage& msg) -> void; + + /// @brief Handles an Order Executed With Price message: reduces or removes + /// the referenced order by its executed share quantity. + /// + /// @param msg The parsed Order Executed With Price message. auto handle_message(const OrderExecutedWithPriceMessage& msg) -> void; + + /// @brief Handles an Order Cancel message: reduces or removes the + /// referenced order by its cancelled share quantity. + /// + /// @param msg The parsed Order Cancel message. auto handle_message(const OrderCancelMessage& msg) -> void; + + /// @brief Handles an Order Delete message: removes the referenced order + /// from the book entirely. + /// + /// @param msg The parsed Order Delete message. auto handle_message(const OrderDeleteMessage& msg) -> void; + + /// @brief Handles an Order Replace message: removes the original order and + /// inserts its replacement at the new reference number, quantity, + /// and price. + /// + /// @param msg The parsed Order Replace message. auto handle_message(const OrderReplaceMessage& msg) -> void; - // Generic handler/sink for message types that do not impact book state. + /// @brief Generic handler/sink for message types that do not impact book state. + /// + /// @tparam T The message type being ignored. + /// @param msg The message instance, ignored. template auto handle_message(const T& /* msg */) -> void { /* No-op for irrelevant messages */ } /// @brief Creates an Order object and inserts it into the appropriate PriceLevel. + /// + /// @param order_ref The exchange-assigned order reference number. + /// @param side 'B' for Buy, 'S' for Sell; selects the bid or ask side. + /// @param shares The initial quantity of shares in the order. + /// @param price The limit price of the order. + /// @param stock The stock symbol for this order. auto add_order( uint64_t order_ref, char side, uint32_t shares, uint32_t price, const std::string& stock ) -> void; From 40dfa48e19d298d2c585686fcb5254aa5995b6e0 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:20 +0100 Subject: [PATCH 079/144] docs: add Doxygen docs to overlay.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/overlay.hpp | 128 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 1 deletion(-) diff --git a/include/itch/overlay.hpp b/include/itch/overlay.hpp index 8ee7b1d..6974468 100644 --- a/include/itch/overlay.hpp +++ b/include/itch/overlay.hpp @@ -1,5 +1,15 @@ #pragma once +/// @file +/// @brief Zero-copy typed views over raw ITCH message frames. +/// +/// Provides `MessageView` and a family of per-message-type `*View` subclasses +/// that read fields lazily, converting each field from network byte order on +/// access rather than eagerly decoding into a host-order struct. This contrasts +/// with the eager `Parser` in parser.hpp, which decodes every field up front. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -17,6 +27,12 @@ namespace detail { /// @brief Reads an integral field of width sizeof(T) at `offset`, converting from /// network (big-endian) order on access. +/// +/// @tparam FieldType The type of the field to read. +/// @param base Pointer to the start of the message frame. +/// @param offset Byte offset of the field within the frame. +/// @return The field value, converted from big-endian if it is a multi-byte +/// integral type. template [[nodiscard]] inline auto read_field(const std::byte* base, std::size_t offset) noexcept -> FieldType { @@ -30,6 +46,10 @@ template } /// @brief Reads the 48-bit ITCH timestamp at `offset` into a 64-bit value. +/// +/// @param base Pointer to the start of the message frame. +/// @param offset Byte offset of the timestamp field within the frame. +/// @return The timestamp, in nanoseconds past midnight. [[nodiscard]] inline auto read_timestamp(const std::byte* base, std::size_t offset) noexcept -> std::uint64_t { const auto high = read_field(base, offset); @@ -39,6 +59,10 @@ template } /// @brief Views an 8-byte fixed-width stock field as a string_view (untrimmed). +/// +/// @param base Pointer to the start of the message frame. +/// @param offset Byte offset of the stock field within the frame. +/// @return An untrimmed view of the 8-byte stock symbol field. [[nodiscard]] inline auto read_stock(const std::byte* base, std::size_t offset) noexcept -> std::string_view { const void* raw = base + offset; @@ -63,37 +87,54 @@ constexpr std::size_t TIMESTAMP_OFFSET = 5; /// underlying buffer lives. class MessageView { public: + /// @brief Constructs an empty view with no underlying frame. constexpr MessageView() noexcept = default; + + /// @brief Constructs a view over a raw ITCH message frame. + /// + /// @param data Pointer to the start of the frame (the type byte). + /// @param size Size of the frame in bytes. explicit MessageView(const std::byte* data, std::size_t size) noexcept : m_data {data}, m_size {size} {} /// @brief The one-byte message type. + /// @return The one-byte message type. [[nodiscard]] auto type() const noexcept -> char { return static_cast(static_cast(m_data[0])); } /// @brief The locate code identifying the security. + /// @return The locate code identifying the security. [[nodiscard]] auto stock_locate() const noexcept -> std::uint16_t { return detail::read_field(m_data, detail::STOCK_LOCATE_OFFSET); } /// @brief The Nasdaq internal tracking number. + /// @return The Nasdaq internal tracking number. [[nodiscard]] auto tracking_number() const noexcept -> std::uint16_t { return detail::read_field(m_data, detail::TRACKING_NUMBER_OFFSET); } /// @brief The message timestamp (nanoseconds past midnight). + /// @return The message timestamp (nanoseconds past midnight). [[nodiscard]] auto timestamp() const noexcept -> std::uint64_t { return detail::read_timestamp(m_data, detail::TIMESTAMP_OFFSET); } /// @brief The raw frame size in bytes. + /// @return The raw frame size in bytes. [[nodiscard]] auto size() const noexcept -> std::size_t { return m_size; } /// @brief Pointer to the raw frame (the type byte). + /// @return Pointer to the raw frame (the type byte). [[nodiscard]] auto data() const noexcept -> const std::byte* { return m_data; } /// @brief Reads an arbitrary integral field at a byte offset, big-endian. + /// + /// @tparam FieldType The type of the field to read. + /// @param offset Byte offset of the field within the frame. + /// @return The field value, converted from big-endian if it is a multi-byte + /// integral type. template [[nodiscard]] auto read(std::size_t offset) const noexcept -> FieldType { return detail::read_field(m_data, offset); @@ -108,14 +149,29 @@ class MessageView { class AddOrderView : public MessageView { public: using MessageView::MessageView; + + /// @brief Day-unique reference number for the order. + /// @return Day-unique reference number for the order. [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { return read(11); } + + /// @brief 'B' buy, 'S' sell. + /// @return 'B' buy, 'S' sell. [[nodiscard]] auto buy_sell_indicator() const noexcept -> char { return read(19); } + + /// @brief Displayed share quantity. + /// @return Displayed share quantity. [[nodiscard]] auto shares() const noexcept -> std::uint32_t { return read(20); } + + /// @brief Stock symbol, right padded with spaces. + /// @return Stock symbol, right padded with spaces. [[nodiscard]] auto stock() const noexcept -> std::string_view { return detail::read_stock(m_data, 24); } + + /// @brief Display price (4 implied decimals). + /// @return Display price (4 implied decimals). [[nodiscard]] auto price() const noexcept -> std::uint32_t { return read(32); } }; @@ -123,12 +179,21 @@ class AddOrderView : public MessageView { class OrderExecutedView : public MessageView { public: using MessageView::MessageView; + + /// @brief Reference number of the executed order. + /// @return Reference number of the executed order. [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { return read(11); } + + /// @brief Number of shares executed. + /// @return Number of shares executed. [[nodiscard]] auto executed_shares() const noexcept -> std::uint32_t { return read(19); } + + /// @brief Day-unique match number for the execution. + /// @return Day-unique match number for the execution. [[nodiscard]] auto match_number() const noexcept -> std::uint64_t { return read(23); } @@ -138,16 +203,31 @@ class OrderExecutedView : public MessageView { class OrderExecutedWithPriceView : public MessageView { public: using MessageView::MessageView; + + /// @brief Reference number of the executed order. + /// @return Reference number of the executed order. [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { return read(11); } + + /// @brief Number of shares executed. + /// @return Number of shares executed. [[nodiscard]] auto executed_shares() const noexcept -> std::uint32_t { return read(19); } + + /// @brief Day-unique match number for the execution. + /// @return Day-unique match number for the execution. [[nodiscard]] auto match_number() const noexcept -> std::uint64_t { return read(23); } + + /// @brief 'Y' if the trade is printable to the tape, else 'N'. + /// @return 'Y' if the trade is printable to the tape, else 'N'. [[nodiscard]] auto printable() const noexcept -> char { return read(31); } + + /// @brief Price at which the order executed (4 decimals). + /// @return Price at which the order executed (4 decimals). [[nodiscard]] auto execution_price() const noexcept -> std::uint32_t { return read(32); } @@ -157,9 +237,15 @@ class OrderExecutedWithPriceView : public MessageView { class OrderCancelView : public MessageView { public: using MessageView::MessageView; + + /// @brief Reference number of the cancelled order. + /// @return Reference number of the cancelled order. [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { return read(11); } + + /// @brief Number of shares cancelled. + /// @return Number of shares cancelled. [[nodiscard]] auto cancelled_shares() const noexcept -> std::uint32_t { return read(19); } @@ -169,6 +255,9 @@ class OrderCancelView : public MessageView { class OrderDeleteView : public MessageView { public: using MessageView::MessageView; + + /// @brief Reference number of the deleted order. + /// @return Reference number of the deleted order. [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { return read(11); } @@ -178,13 +267,25 @@ class OrderDeleteView : public MessageView { class OrderReplaceView : public MessageView { public: using MessageView::MessageView; + + /// @brief Reference number being replaced. + /// @return Reference number being replaced. [[nodiscard]] auto original_order_reference_number() const noexcept -> std::uint64_t { return read(11); } + + /// @brief New reference number for the order. + /// @return New reference number for the order. [[nodiscard]] auto new_order_reference_number() const noexcept -> std::uint64_t { return read(19); } + + /// @brief New displayed share quantity. + /// @return New displayed share quantity. [[nodiscard]] auto shares() const noexcept -> std::uint32_t { return read(27); } + + /// @brief New display price (4 implied decimals). + /// @return New display price (4 implied decimals). [[nodiscard]] auto price() const noexcept -> std::uint32_t { return read(31); } }; @@ -192,15 +293,33 @@ class OrderReplaceView : public MessageView { class NonCrossTradeView : public MessageView { public: using MessageView::MessageView; + + /// @brief Reference number of the non-displayed order. + /// @return Reference number of the non-displayed order. [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { return read(11); } + + /// @brief 'B' buy, 'S' sell. + /// @return 'B' buy, 'S' sell. [[nodiscard]] auto buy_sell_indicator() const noexcept -> char { return read(19); } + + /// @brief Number of shares traded. + /// @return Number of shares traded. [[nodiscard]] auto shares() const noexcept -> std::uint32_t { return read(20); } + + /// @brief Stock symbol, right padded with spaces. + /// @return Stock symbol, right padded with spaces. [[nodiscard]] auto stock() const noexcept -> std::string_view { return detail::read_stock(m_data, 24); } + + /// @brief Trade price (4 implied decimals). + /// @return Trade price (4 implied decimals). [[nodiscard]] auto price() const noexcept -> std::uint32_t { return read(32); } + + /// @brief Day-unique match number for the trade. + /// @return Day-unique match number for the trade. [[nodiscard]] auto match_number() const noexcept -> std::uint64_t { return read(36); } @@ -211,6 +330,9 @@ using ViewCallback = std::function; /// @brief Builds the per-type expected wire-size table at compile time, mirroring /// the eager parser's so the overlay validates frame lengths identically. +/// +/// @return An array indexed by message-type byte, holding the expected wire size +/// for each known message type (zero for unknown types). [[nodiscard]] consteval auto build_size_table() -> std::array { std::array table {}; auto add = [&table](char type) { @@ -229,7 +351,11 @@ inline constexpr auto SIZE_TABLE = build_size_table(); /// Framing and length validation match `Parser`: the 2-byte length prefix is /// honoured, unknown type bytes and undersized frames are skipped. Unlike the /// eager parser, no fields are decoded; the callback receives a view it can -/// inspect lazily. Returns the number of views delivered. +/// inspect lazily. +/// +/// @param data The raw buffer containing one or more length-prefixed ITCH frames. +/// @param callback Invoked with a `MessageView` for each well-formed frame found. +/// @return The number of views delivered to `callback`. inline auto for_each_message(std::span data, const ViewCallback& callback) -> std::uint64_t { std::uint64_t delivered = 0; From 291d2fa4b0d18a55bbb0674a91e629c9ce9c492b Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:20 +0100 Subject: [PATCH 080/144] docs: add Doxygen docs to parser.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/parser.hpp | 66 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/include/itch/parser.hpp b/include/itch/parser.hpp index 751da38..6cb3e2b 100644 --- a/include/itch/parser.hpp +++ b/include/itch/parser.hpp @@ -1,5 +1,17 @@ #pragma once +/// @file +/// @brief Public parsing interface for NASDAQ TotalView-ITCH 5.0 feeds, plus +/// the low-level byte-unpacking utilities that back it. +/// +/// `Parser` frames and decodes a raw ITCH byte stream into the `Message` +/// variant defined in itch/messages.hpp, offering both throwing and +/// non-throwing (`std::expected`-based) entry points over buffers, spans, and +/// streams. The `itch::utils` helpers underneath handle endianness conversion +/// and fixed-width field extraction, and are also reused by the encoder. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -170,9 +182,18 @@ class Parser { auto parse(std::span data, const MessageCallback& callback) -> void; /// @brief Parses all messages from a byte span and returns them in a vector. + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @return A std::vector containing all parsed messages. + /// @throw std::runtime_error if the buffer ends in the middle of a message. auto parse(std::span data) -> std::vector; /// @brief Parses messages from a byte span, keeping only the requested types. + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @param messages A vector of message type characters to keep (e.g., {'A', 'P'}). + /// @return A std::vector containing only the filtered messages. + /// @throw std::runtime_error if the buffer ends in the middle of a message. auto parse(std::span data, const std::vector& messages) -> std::vector; @@ -196,6 +217,9 @@ class Parser { -> std::expected; /// @brief Non-throwing parse that collects all messages into a vector. + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @return All parsed messages on success, or a `ParseError` describing the failure. [[nodiscard]] auto try_parse(std::span data) -> std::expected, ParseError>; #endif @@ -205,15 +229,23 @@ class Parser { /// Passing an empty function clears any previously installed callback. The /// default behavior, with no callback installed, is to silently skip and /// count the offending frame. + /// + /// @param callback The diagnostics callback to install, or an empty function + /// to clear any previously installed callback. auto set_error_callback(ErrorCallback callback) -> void; /// @brief The number of frames skipped because their type byte was unknown. + /// + /// @return The running count of frames skipped for an unrecognized type byte. [[nodiscard]] auto unknown_message_count() const noexcept -> std::uint64_t { return m_unknown_message_count; } /// @brief The number of frames skipped because their declared length was too /// small for the message type. + /// + /// @return The running count of frames skipped for a declared length too + /// small for their message type. [[nodiscard]] auto malformed_message_count() const noexcept -> std::uint64_t { return m_malformed_message_count; } @@ -226,12 +258,20 @@ class Parser { private: /// @brief The shared, non-throwing framing loop backing every parse overload. - /// Returns `std::nullopt` on success, or the `ParseError` that aborted - /// the loop (only an unrecoverable truncation aborts it). + /// + /// @param data A pointer to the start of the memory buffer containing ITCH data. + /// @param size The total size of the buffer in bytes. + /// @param callback A function to be called for each successfully parsed message. + /// @return `std::nullopt` on success, or the `ParseError` that aborted the + /// loop (only an unrecoverable truncation aborts it). auto parse_impl(const char* data, std::size_t size, const MessageCallback& callback) -> std::optional; /// @brief Records a recoverable framing problem and notifies the callback. + /// + /// @param error The category of problem that occurred. + /// @param message_type The offending message type byte (`'\0'` when not + /// applicable, e.g. for a truncated header). auto report_error(ParseError error, char message_type) -> void; ErrorCallback m_error_callback {}; @@ -247,6 +287,10 @@ namespace utils { /// `std::bit_cast` based reversal. This deliberately avoids reading an inactive /// union member, which is undefined behavior in C++ and a hazard on the parser /// hot path. +/// +/// @tparam IntType The integral type to byte-swap. +/// @param value The value whose byte order should be reversed. +/// @return `value` with its byte order reversed. template [[nodiscard]] constexpr auto swap_bytes(IntType value) noexcept -> IntType { #ifdef __cpp_lib_byteswap @@ -266,6 +310,10 @@ template /// /// On little-endian hosts this swaps the bytes; on big-endian hosts it is a /// no-op. The choice is made at compile time via `std::endian`. +/// +/// @tparam IntType The integral type to convert. +/// @param value The big-endian value to convert. +/// @return `value` converted to host byte order. template [[nodiscard]] constexpr auto from_big_endian(IntType value) noexcept -> IntType { if constexpr (std::endian::native == std::endian::little) { @@ -277,14 +325,28 @@ template /// @brief Unpacks a value of type T from the buffer, advancing the offset. /// Handles endianness conversion for multi-byte integral types. +/// +/// @tparam ValueType The type of value to unpack. +/// @param buffer The buffer to read from. +/// @param offset The byte offset to read from, advanced past the unpacked value. +/// @return The unpacked value, converted to host byte order when applicable. template auto unpack(const char* buffer, std::size_t& offset) -> ValueType; /// @brief Copies a fixed-width character field out of the buffer. +/// +/// @param buffer The buffer to read from. +/// @param offset The byte offset to read from, advanced past the copied field. +/// @param dest The destination buffer to copy the field into. +/// @param size The number of characters to copy. inline auto unpack_string(const char* buffer, std::size_t& offset, char* dest, std::size_t size) -> void; /// @brief Unpacks a 48-bit big-endian ITCH timestamp (2-byte high, 4-byte low). +/// +/// @param buffer The buffer to read from. +/// @param offset The byte offset to read from, advanced past the timestamp. +/// @return The timestamp as nanoseconds past midnight. inline auto unpack_timestamp(const char* buffer, std::size_t& offset) -> std::uint64_t; } // namespace utils From b4c4993df4ca7c93a1d6db85a684ba80a4c4f1be Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:20 +0100 Subject: [PATCH 081/144] docs: add Doxygen docs to price.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/price.hpp | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/include/itch/price.hpp b/include/itch/price.hpp index afd33a0..3a93480 100644 --- a/include/itch/price.hpp +++ b/include/itch/price.hpp @@ -1,5 +1,14 @@ #pragma once +/// @file +/// @brief Strongly typed, fixed-point price representation for ITCH price fields. +/// +/// Wraps the raw on-wire integer price in a type that carries its own decimal +/// scale, so standard 4-decimal prices and MWCB 8-decimal decline-level prices +/// cannot be mixed, compared, or formatted with the wrong divisor by accident. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -32,15 +41,23 @@ class BasicPrice { /// @brief The number of implied decimal places for this price scale. static constexpr unsigned int decimals = Decimals; + /// @brief Constructs a zero-valued price at this scale. constexpr BasicPrice() noexcept = default; /// @brief Wraps a raw on-wire price value at this scale. + /// + /// @param raw_value The raw integer price, exactly as it appears on the wire. explicit constexpr BasicPrice(raw_type raw_value) noexcept : m_raw {raw_value} {} /// @brief The underlying raw integer value, exactly as it appears on the wire. + /// + /// @return The raw integer price at this scale. [[nodiscard]] constexpr auto raw() const noexcept -> raw_type { return m_raw; } /// @brief The scale's divisor, i.e. 10^Decimals. + /// + /// @return The divisor used to convert the raw integer value into a + /// floating-point or decimal-string representation. [[nodiscard]] static constexpr auto divisor() noexcept -> raw_type { raw_type result {1}; for (unsigned int exponent {0}; exponent < Decimals; ++exponent) { @@ -50,17 +67,31 @@ class BasicPrice { } /// @brief The price as a floating-point value (raw / 10^Decimals). + /// + /// @return The price converted to a `double`. [[nodiscard]] constexpr auto to_double() const noexcept -> double { return static_cast(m_raw) / static_cast(divisor()); } /// @brief The price as a fixed-precision decimal string. + /// + /// @return The price formatted with exactly `Decimals` fractional digits. [[nodiscard]] auto to_string() const -> std::string { return std::format("{:.{}f}", to_double(), Decimals); } + /// @brief Compares two prices of the same scale for equality by raw value. + /// + /// @param lhs The left-hand price. + /// @param rhs The right-hand price. + /// @return `true` if both prices have the same raw value. [[nodiscard]] friend constexpr auto operator==(BasicPrice, BasicPrice) noexcept -> bool = default; + /// @brief Orders two prices of the same scale by raw value. + /// + /// @param lhs The left-hand price. + /// @param rhs The right-hand price. + /// @return The three-way comparison result between `lhs` and `rhs`. [[nodiscard]] friend constexpr auto operator<=>(BasicPrice, BasicPrice) noexcept = default; private: @@ -74,11 +105,17 @@ using StandardPrice = BasicPrice; using MwcbPrice = BasicPrice; /// @brief Wraps a raw 4-decimal price value in the typed StandardPrice. +/// +/// @param raw_value The raw on-wire 4-decimal price. +/// @return The value wrapped as a `StandardPrice`. [[nodiscard]] constexpr auto make_price(std::uint32_t raw_value) noexcept -> StandardPrice { return StandardPrice {raw_value}; } /// @brief Wraps a raw 8-decimal MWCB price value in the typed MwcbPrice. +/// +/// @param raw_value The raw on-wire 8-decimal MWCB decline-level price. +/// @return The value wrapped as an `MwcbPrice`. [[nodiscard]] constexpr auto make_mwcb_price(std::uint64_t raw_value) noexcept -> MwcbPrice { return MwcbPrice {raw_value}; } @@ -88,6 +125,12 @@ using MwcbPrice = BasicPrice; /// @brief Formats a price with its scale's number of decimals, e.g. "150.0000". template struct std::formatter> : std::formatter { + /// @brief Formats `price` as a fixed-precision decimal string. + /// + /// @param price The price to format. + /// @param ctx The format context to write the formatted output into. + /// @return An output iterator positioned after the formatted text, per the + /// `std::formatter` contract. auto format(itch::BasicPrice price, std::format_context& ctx) const { return std::formatter::format(price.to_double(), ctx); } From 539b9f95a8f3a87c55c44e5f0df7c0a822a0d8ee Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:21 +0100 Subject: [PATCH 082/144] docs: add Doxygen docs to replay.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/replay.hpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/include/itch/replay.hpp b/include/itch/replay.hpp index 5009b54..7504561 100644 --- a/include/itch/replay.hpp +++ b/include/itch/replay.hpp @@ -1,5 +1,14 @@ #pragma once +/// @file +/// @brief A timestamp-paced replay engine for previously captured ITCH streams. +/// +/// Wraps the `Parser` so a consumer can play back a buffer at (a multiple of) +/// its original wall-clock cadence rather than as fast as the CPU can parse +/// it, which is useful for realistic backtesting and system simulation. +/// +/// @author Bertin Balouki SIMYELI + #include #include @@ -26,16 +35,23 @@ class ReplayEngine { : m_speed_multiplier {speed_multiplier} {} /// @brief Sets the speed multiplier (see the constructor). + /// + /// @param speed_multiplier Wall-clock speed relative to the original feed + /// (see the constructor for the exact semantics). auto set_speed(double speed_multiplier) noexcept -> void { m_speed_multiplier = speed_multiplier; } /// @brief The current speed multiplier. + /// + /// @return The currently configured speed multiplier. [[nodiscard]] auto speed() const noexcept -> double { return m_speed_multiplier; } /// @brief Parses `data` and invokes `callback` for each message, paced by the /// message timestamps. /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @param callback A function to be called for each successfully parsed message. /// @return The number of messages replayed. auto replay(std::span data, const MessageCallback& callback) -> std::uint64_t; From d876ed79eac129439a24cd4012996a080e29112d Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:21 +0100 Subject: [PATCH 083/144] docs: add Doxygen docs to tape.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/tape.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/itch/tape.hpp b/include/itch/tape.hpp index 6b4378a..b294861 100644 --- a/include/itch/tape.hpp +++ b/include/itch/tape.hpp @@ -1,5 +1,15 @@ #pragma once +/// @file +/// @brief A normalized trade record and callback type for consuming the ITCH +/// trade tape. +/// +/// Execution-related ITCH message types are heterogeneous in shape; this +/// header defines the single, flattened `Trade` record that callers consume +/// instead of switching over the raw message variant themselves. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include From 7bad568992b26a3eb37025e3be8582c8edb3f59c Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:21 +0100 Subject: [PATCH 084/144] docs: add Doxygen docs to time.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/time.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/itch/time.hpp b/include/itch/time.hpp index 3976c22..fdc2e0c 100644 --- a/include/itch/time.hpp +++ b/include/itch/time.hpp @@ -1,5 +1,15 @@ #pragma once +/// @file +/// @brief Helpers for converting raw ITCH nanoseconds-past-midnight timestamps +/// into absolute time points and human-readable strings. +/// +/// ITCH timestamps carry no date or time-zone of their own, so every +/// conversion here takes the session date as an explicit parameter and +/// performs no time-zone adjustment. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include From 6bdc96d095400ef7ddf2281b1702008b50cd3847 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:22 +0100 Subject: [PATCH 085/144] docs: add Doxygen docs to moldudp64.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/transport/moldudp64.hpp | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/include/itch/transport/moldudp64.hpp b/include/itch/transport/moldudp64.hpp index 263e103..b979e79 100644 --- a/include/itch/transport/moldudp64.hpp +++ b/include/itch/transport/moldudp64.hpp @@ -1,5 +1,15 @@ #pragma once +/// @file +/// @brief Decoder for NASDAQ's MoldUDP64 UDP multicast market-data framing. +/// +/// This header declares the `MoldUdp64Header` wire structure and the +/// `MoldUdp64Decoder` that strips it from each datagram and feeds the +/// contained, length-prefixed ITCH message blocks through the shared +/// `Parser`. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -30,6 +40,7 @@ struct MoldUdp64Header { static constexpr std::uint16_t END_OF_SESSION = 0xFFFF; /// @brief The session id as a view with trailing padding removed. + /// @return A view of the session id with trailing spaces/NULs stripped. [[nodiscard]] auto session_view() const -> std::string_view { std::size_t length = session.size(); while (length > 0 && (session[length - 1] == ' ' || session[length - 1] == '\0')) { @@ -39,9 +50,11 @@ struct MoldUdp64Header { } /// @brief A heartbeat carries no message blocks (`message_count == 0`). + /// @return True if the packet is a heartbeat, false otherwise. [[nodiscard]] auto is_heartbeat() const noexcept -> bool { return message_count == 0; } /// @brief End-of-session is signalled by the sentinel `message_count`. + /// @return True if the packet signals end of session, false otherwise. [[nodiscard]] auto is_end_of_session() const noexcept -> bool { return message_count == END_OF_SESSION; } @@ -61,6 +74,8 @@ class MoldUdp64Decoder { static constexpr std::size_t HEADER_SIZE = 20; /// @brief Constructs a decoder that calls `callback` for each ITCH message. + /// @param callback Invoked with each ITCH message decoded from a data + /// packet's message blocks. explicit MoldUdp64Decoder(MessageCallback callback); /// @brief Decodes a single MoldUDP64 datagram. @@ -71,15 +86,20 @@ class MoldUdp64Decoder { auto decode_packet(std::span packet) -> std::optional; /// @brief The embedded sequence tracker (install gap callbacks here). + /// @return Reference to the embedded `SequenceTracker`. [[nodiscard]] auto tracker() noexcept -> SequenceTracker& { return m_tracker; } + /// @brief The embedded sequence tracker (install gap callbacks here). + /// @return Const reference to the embedded `SequenceTracker`. [[nodiscard]] auto tracker() const noexcept -> const SequenceTracker& { return m_tracker; } /// @brief Total number of datagrams passed to `decode_packet`. + /// @return The count of datagrams decoded so far. [[nodiscard]] auto packets_decoded() const noexcept -> std::uint64_t { return m_packets_decoded; } /// @brief Total number of ITCH messages decoded from all datagrams. + /// @return The total count of decoded ITCH messages. [[nodiscard]] auto messages_decoded() const noexcept -> std::uint64_t { return m_messages_decoded; } From 7c41a48b663c6ffaadba760b560e5c4955bfb25e Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:22 +0100 Subject: [PATCH 086/144] docs: add Doxygen docs to pcap.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/transport/pcap.hpp | 46 ++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/include/itch/transport/pcap.hpp b/include/itch/transport/pcap.hpp index 6b2f3d6..ceaf52c 100644 --- a/include/itch/transport/pcap.hpp +++ b/include/itch/transport/pcap.hpp @@ -1,5 +1,16 @@ #pragma once +/// @file +/// @brief In-house reader for classic pcap and pcapng capture files carrying +/// MoldUDP64/ITCH traffic. +/// +/// This header declares `PcapReader`, which walks the Ethernet/IPv4/IPv6/UDP +/// layers of each captured frame, extracts matching UDP payloads, and feeds +/// them through an embedded `MoldUdp64Decoder` to produce decoded ITCH +/// messages, without depending on libpcap. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -29,6 +40,9 @@ class PcapReader { public: /// @brief Constructs a reader that forwards each decoded ITCH message to /// `callback`. + /// + /// @param callback Invoked with each ITCH message decoded from the + /// capture. explicit PcapReader(MessageCallback callback); /// @brief Decodes an in-memory capture buffer. @@ -40,38 +54,62 @@ class PcapReader { /// @brief Reads and decodes a capture file from disk. /// + /// @param path Filesystem path to the `.pcap` or `.pcapng` file to read. /// @return `true` on a recognized, readable file; `false` if the file cannot /// be opened or is not a capture. auto read_file(const std::string& path) -> bool; /// @brief Restricts decoding to UDP datagrams sent to this destination port. /// When unset, datagrams on any port are decoded. + /// + /// @param port The destination UDP port to accept. auto set_udp_port_filter(std::uint16_t port) -> void { m_port_filter = port; } /// @brief Clears any destination-port filter previously installed. auto clear_udp_port_filter() noexcept -> void { m_port_filter = std::nullopt; } /// @brief The embedded MoldUDP64 decoder (sequence tracking lives here). + /// @return Reference to the embedded `MoldUdp64Decoder`. [[nodiscard]] auto mold_decoder() noexcept -> MoldUdp64Decoder& { return m_mold; } + /// @brief The embedded MoldUDP64 decoder (sequence tracking lives here). + /// @return Const reference to the embedded `MoldUdp64Decoder`. [[nodiscard]] auto mold_decoder() const noexcept -> const MoldUdp64Decoder& { return m_mold; } /// @brief Number of UDP datagrams extracted and handed to the MoldUDP64 /// decoder. + /// @return The count of UDP datagrams processed so far. [[nodiscard]] auto udp_datagrams() const noexcept -> std::uint64_t { return m_udp_datagrams; } /// @brief Total ITCH messages decoded across the whole capture. + /// @return The total count of decoded ITCH messages. [[nodiscard]] auto messages_decoded() const noexcept -> std::uint64_t { return m_mold.messages_decoded(); } private: - // Container-format readers; each returns false if the buffer is not that - // format or is structurally invalid. + /// @brief Parses the buffer as a classic pcap file and decodes its frames. + /// + /// @param capture The full contents of the candidate capture file. + /// @param swapped Whether the file's magic number indicated byte-swapped + /// (opposite-endian) header fields. + /// @return `true` if the buffer was a valid classic pcap file, `false` if + /// it is not that format or is structurally invalid. auto read_classic_pcap(std::span capture, bool swapped) -> bool; + + /// @brief Parses the buffer as a pcapng file and decodes its frames. + /// + /// @param capture The full contents of the candidate capture file. + /// @return `true` if the buffer was a valid pcapng file, `false` if it is + /// not that format or is structurally invalid. auto read_pcapng(std::span capture) -> bool; - // Walks a single captured frame of the given link type and, if it carries a - // matching UDP datagram, feeds the payload to the MoldUDP64 decoder. + /// @brief Walks a single captured frame of the given link type and, if it + /// carries a matching UDP datagram, feeds the payload to the + /// MoldUDP64 decoder. + /// + /// @param frame The raw captured frame bytes (link-layer through payload). + /// @param link_type The capture's link-layer type (for example Ethernet), + /// identifying how to interpret `frame`. auto handle_frame(std::span frame, std::uint32_t link_type) -> void; MoldUdp64Decoder m_mold; From c62493fb3ae816b4de92c4a92945e342be9d1f96 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:22 +0100 Subject: [PATCH 087/144] docs: add Doxygen docs to sequencing.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/transport/sequencing.hpp | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/include/itch/transport/sequencing.hpp b/include/itch/transport/sequencing.hpp index afe2eb3..7cf3011 100644 --- a/include/itch/transport/sequencing.hpp +++ b/include/itch/transport/sequencing.hpp @@ -1,5 +1,15 @@ #pragma once +/// @file +/// @brief Per-session sequence-gap detection shared by the MoldUDP64 and +/// SoupBinTCP transport decoders. +/// +/// This header declares the `RetransmitRequester` hook a caller implements to +/// drive gap recovery, and the `SequenceTracker` that watches per-packet +/// sequence numbers and reports gaps through a callback and/or the requester. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -19,15 +29,31 @@ namespace itch::transport { /// feeding the recovered messages back through the decoder. class RetransmitRequester { public: + /// @brief Constructs a requester with no state. RetransmitRequester() = default; + /// @brief Copy-constructs a requester. + /// @param other The requester to copy. RetransmitRequester(const RetransmitRequester&) = default; + /// @brief Move-constructs a requester. + /// @param other The requester to move from. RetransmitRequester(RetransmitRequester&&) noexcept = default; + /// @brief Copy-assigns a requester. + /// @param other The requester to copy. + /// @return Reference to this requester. auto operator=(const RetransmitRequester&) -> RetransmitRequester& = default; + /// @brief Move-assigns a requester. + /// @param other The requester to move from. + /// @return Reference to this requester. auto operator=(RetransmitRequester&&) noexcept -> RetransmitRequester& = default; + /// @brief Destroys the requester. virtual ~RetransmitRequester() = default; /// @brief Requests retransmission of `count` messages starting at sequence /// `start_sequence` for the given session. + /// + /// @param session The session identifier the gap was observed on. + /// @param start_sequence The first missing sequence number to retransmit. + /// @param count The number of missing messages to retransmit. virtual auto request_retransmit( std::string_view session, std::uint64_t start_sequence, std::uint64_t count ) -> void = 0; @@ -52,6 +78,10 @@ class SequenceTracker { /// @brief Records that a packet beginning at `first_sequence` carried `count` /// sequenced messages for `session`. /// + /// @param session The session identifier the packet belongs to. + /// @param first_sequence The sequence number of the first message in the + /// packet. + /// @param count The number of sequenced messages carried by the packet. /// @return The number of missing messages detected (0 when in order). auto observe(std::string_view session, std::uint64_t first_sequence, std::uint64_t count) -> std::uint64_t { @@ -89,14 +119,20 @@ class SequenceTracker { } /// @brief Installs the gap-notification callback (empty clears it). + /// @param callback Invoked once per detected gap. auto set_gap_callback(GapCallback callback) -> void { m_gap_callback = std::move(callback); } /// @brief Installs a non-owning retransmission hook (nullptr clears it). + /// @param requester Non-owning pointer to the requester to notify on + /// gaps, or nullptr to clear it. auto set_retransmit_requester(RetransmitRequester* requester) noexcept -> void { m_requester = requester; } /// @brief The next sequence number expected for a session, if it has been seen. + /// @param session The session identifier to look up. + /// @return The next expected sequence number, or `std::nullopt` if the + /// session has not been observed. [[nodiscard]] auto expected_next(std::string_view session) const -> std::optional { const auto iter = m_expected_by_session.find(std::string {session}); @@ -107,9 +143,11 @@ class SequenceTracker { } /// @brief Total number of missing messages detected across all sessions. + /// @return The total gap count. [[nodiscard]] auto gap_count() const noexcept -> std::uint64_t { return m_gap_count; } /// @brief Total number of distinct sequenced messages observed. + /// @return The total count of distinct messages observed. [[nodiscard]] auto messages_seen() const noexcept -> std::uint64_t { return m_messages_seen; } /// @brief Forgets all per-session state and resets the counters. From d38667962e3b62a35f7d5c6b534f1bc57a02cf49 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:23 +0100 Subject: [PATCH 088/144] docs: add Doxygen docs to soupbintcp.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/transport/soupbintcp.hpp | 38 +++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/include/itch/transport/soupbintcp.hpp b/include/itch/transport/soupbintcp.hpp index d20d552..b80c790 100644 --- a/include/itch/transport/soupbintcp.hpp +++ b/include/itch/transport/soupbintcp.hpp @@ -1,5 +1,16 @@ #pragma once +/// @file +/// @brief Stateful decoder for the SoupBinTCP session-layer protocol. +/// +/// SoupBinTCP is NASDAQ's reliable, ordered TCP framing used for the Glimpse +/// snapshot service and for sequenced/unsequenced ITCH message delivery and +/// replay. This header declares the packet type enumeration and the +/// `SoupBinDecoder` that reassembles a raw byte stream into discrete packets +/// and decoded ITCH messages. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -50,34 +61,57 @@ class SoupBinDecoder { std::function payload)>; /// @brief Constructs a decoder that calls `callback` for each ITCH message. + /// + /// @param callback Invoked with each ITCH message decoded from a + /// sequenced or unsequenced data packet. explicit SoupBinDecoder(MessageCallback callback); /// @brief Feeds a chunk of the TCP byte stream, processing any complete /// packets it completes. + /// + /// @param bytes The next contiguous chunk of bytes read from the TCP + /// connection. auto feed(std::span bytes) -> void; /// @brief Installs the control-packet event callback (empty clears it). + /// + /// @param callback Invoked for each non-data control packet decoded. auto set_event_callback(EventCallback callback) -> void; /// @brief The embedded sequence tracker (install gap callbacks here). + /// @return Reference to the embedded `SequenceTracker`. [[nodiscard]] auto tracker() noexcept -> SequenceTracker& { return m_tracker; } + /// @brief The embedded sequence tracker (install gap callbacks here). + /// @return Const reference to the embedded `SequenceTracker`. [[nodiscard]] auto tracker() const noexcept -> const SequenceTracker& { return m_tracker; } /// @brief The session id learned from Login Accepted (empty until then). + /// @return The current session id, or an empty view before login. [[nodiscard]] auto current_session() const -> std::string_view { return m_session; } /// @brief The next sequence number the decoder expects for a data packet. + /// @return The next expected sequence number. [[nodiscard]] auto next_sequence() const noexcept -> std::uint64_t { return m_next_sequence; } /// @brief Total sequenced/unsequenced data messages decoded. + /// @return The total count of decoded data messages. [[nodiscard]] auto messages_decoded() const noexcept -> std::uint64_t { return m_messages_decoded; } private: - // Decodes the single packet [type byte + payload] and dispatches it. + /// @brief Decodes the single packet (type byte plus payload) and + /// dispatches it to the event callback or the application decoder. + /// + /// @param type The one-byte SoupBinTCP packet type. + /// @param payload The packet body following the type byte. auto process_packet(SoupBinPacketType type, std::span payload) -> void; - // Decodes one application message payload through the parser. + + /// @brief Decodes one application message payload through the parser and + /// advances the expected sequence number. + /// + /// @param payload The raw ITCH message bytes carried by a sequenced or + /// unsequenced data packet. auto decode_application_message(std::span payload) -> void; Parser m_parser {}; From bed18095bffd3154a645daf8e2d85f3da01bf593 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:23 +0100 Subject: [PATCH 089/144] docs: add Doxygen docs to venue.hpp Co-Authored-By: Claude Sonnet 5 --- include/itch/venue.hpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/include/itch/venue.hpp b/include/itch/venue.hpp index 97bd556..fea9f10 100644 --- a/include/itch/venue.hpp +++ b/include/itch/venue.hpp @@ -1,5 +1,15 @@ #pragma once +/// @file +/// @brief The extension point for adding new ITCH-like venues/protocol versions +/// alongside NASDAQ TotalView-ITCH 5.0. +/// +/// Defines the `VenuePolicy` concept that a venue policy type must satisfy, and +/// `Nasdaq50`, the concrete policy for the only protocol this library +/// implements today. +/// +/// @author Bertin Balouki SIMYELI + #include #include @@ -29,11 +39,18 @@ concept VenuePolicy = requires { /// implemented today). New venues are added by writing another policy of /// the same shape and parameterizing the dispatch builder on it. struct Nasdaq50 { + /// @brief The identifying name of this venue/protocol version. + /// + /// @return The human-readable venue/version name. [[nodiscard]] static constexpr auto name() noexcept -> std::string_view { return "NASDAQ TotalView-ITCH 5.0"; } /// @brief Enumerates this venue's message-type registry. + /// + /// @tparam Visitor The visitor type; must be callable as + /// `visitor.template operator()(char)`. + /// @param visitor The visitor invoked once per message type. template static constexpr auto for_each_message_type(Visitor&& visitor) -> void { detail::for_each_message_type(static_cast(visitor)); From cf4c8d144172ff629b4a844836ed0aadb8c21298 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:23 +0100 Subject: [PATCH 090/144] docs: add Doxygen docs to frame_builders.hpp Co-Authored-By: Claude Sonnet 5 --- tests/transport/frame_builders.hpp | 61 ++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/tests/transport/frame_builders.hpp b/tests/transport/frame_builders.hpp index 1fdb42b..b3bc6ae 100644 --- a/tests/transport/frame_builders.hpp +++ b/tests/transport/frame_builders.hpp @@ -1,5 +1,17 @@ #pragma once +/// @file +/// @brief Test-only fixture-building helpers that construct synthetic +/// ITCH/MoldUDP64/SoupBinTCP/pcap frames for unit tests. +/// +/// These helpers build raw ITCH message payloads and wrap them in the +/// MoldUDP64, SoupBinTCP, and pcap envelopes entirely from in-memory bytes, +/// so the transport decoders can be exercised in tests without any real +/// capture files. This is a test fixture helper, not part of the public +/// library API. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -8,11 +20,12 @@ #include "itch/parser.hpp" -// Shared helpers for the transport tests: build raw ITCH frames and wrap them in -// the MoldUDP64 / SoupBinTCP / pcap envelopes from in-memory bytes, so the -// decoders can be exercised without any real capture files. namespace itch::test { +/// @brief Packs `value` into 2 bytes in big-endian order and appends them to +/// `out`. +/// @param out The byte buffer to append to. +/// @param value The 16-bit value to pack. inline auto append_be16(std::vector& out, std::uint16_t value) -> void { const auto big = itch::utils::from_big_endian(value); std::array bytes {}; @@ -20,6 +33,10 @@ inline auto append_be16(std::vector& out, std::uint16_t value) -> voi out.insert(out.end(), bytes.begin(), bytes.end()); } +/// @brief Packs `value` into 4 bytes in big-endian order and appends them to +/// `out`. +/// @param out The byte buffer to append to. +/// @param value The 32-bit value to pack. inline auto append_be32(std::vector& out, std::uint32_t value) -> void { const auto big = itch::utils::from_big_endian(value); std::array bytes {}; @@ -27,6 +44,10 @@ inline auto append_be32(std::vector& out, std::uint32_t value) -> voi out.insert(out.end(), bytes.begin(), bytes.end()); } +/// @brief Packs `value` into 8 bytes in big-endian order and appends them to +/// `out`. +/// @param out The byte buffer to append to. +/// @param value The 64-bit value to pack. inline auto append_be64(std::vector& out, std::uint64_t value) -> void { const auto big = itch::utils::from_big_endian(value); std::array bytes {}; @@ -34,17 +55,29 @@ inline auto append_be64(std::vector& out, std::uint64_t value) -> voi out.insert(out.end(), bytes.begin(), bytes.end()); } +/// @brief Packs `value` into 4 bytes in little-endian order (a direct copy of +/// the in-memory representation, assuming a little-endian host) and +/// appends them to `out`. +/// @param out The byte buffer to append to. +/// @param value The 32-bit value to pack. inline auto append_le32(std::vector& out, std::uint32_t value) -> void { std::array bytes {}; std::memcpy(bytes.data(), &value, sizeof(value)); out.insert(out.end(), bytes.begin(), bytes.end()); } +/// @brief Appends the raw contents of `src` to `out`. +/// @param out The byte buffer to append to. +/// @param src The bytes to append. inline auto append_bytes(std::vector& out, const std::vector& src) -> void { out.insert(out.end(), src.begin(), src.end()); } /// @brief Builds the raw payload (no length prefix) of a System Event message. +/// @param timestamp The 48-bit event timestamp (nanoseconds since midnight) +/// to encode. +/// @param event_code The single-character system event code. +/// @return The encoded System Event message payload. inline auto system_event_payload(std::uint64_t timestamp, char event_code) -> std::vector { std::vector payload; @@ -61,6 +94,13 @@ inline auto system_event_payload(std::uint64_t timestamp, char event_code) /// @brief Builds the raw payload (no length prefix) of an Add Order (`A`) /// message. +/// @param locate The stock locate code. +/// @param ref The order reference number. +/// @param side The single-character side code (`'B'` or `'S'`). +/// @param shares The number of shares in the order. +/// @param symbol The ticker symbol (right-padded/truncated to 8 characters). +/// @param price The raw (unscaled) limit price. +/// @return The encoded Add Order message payload. inline auto add_order_payload( std::uint16_t locate, std::uint64_t ref, char side, std::uint32_t shares, std::string_view symbol, std::uint32_t price @@ -85,6 +125,8 @@ inline auto add_order_payload( /// @brief Wraps a raw payload as a length-prefixed ITCH frame (and message /// block, since both use the same 2-byte big-endian length prefix). +/// @param payload The raw payload to prefix. +/// @return The payload prefixed with its 2-byte big-endian length. inline auto length_prefixed(const std::vector& payload) -> std::vector { std::vector frame; append_be16(frame, static_cast(payload.size())); @@ -94,6 +136,11 @@ inline auto length_prefixed(const std::vector& payload) -> std::vecto /// @brief Builds a complete MoldUDP64 datagram from a list of raw message /// payloads. +/// @param session The session identifier (right-padded/truncated to 10 +/// characters). +/// @param sequence The sequence number of the first message block. +/// @param payloads The raw message payloads to include, in order. +/// @return The encoded MoldUDP64 datagram. inline auto moldudp64_packet( const std::string& session, std::uint64_t sequence, @@ -116,6 +163,9 @@ inline auto moldudp64_packet( } /// @brief Builds a SoupBinTCP packet (2-byte length + type byte + payload). +/// @param type The single-character SoupBinTCP packet type. +/// @param payload The packet body following the type byte. +/// @return The encoded SoupBinTCP packet. inline auto soupbin_packet(char type, const std::vector& payload) -> std::vector { std::vector packet; @@ -126,6 +176,9 @@ inline auto soupbin_packet(char type, const std::vector& payload) } /// @brief Wraps a UDP payload in Ethernet/IPv4/UDP headers (a single frame). +/// @param dst_port The destination UDP port to encode in the UDP header. +/// @param payload The UDP payload to wrap. +/// @return The encoded Ethernet/IPv4/UDP frame. inline auto ethernet_ipv4_udp_frame(std::uint16_t dst_port, const std::vector& payload) -> std::vector { std::vector frame; @@ -161,6 +214,8 @@ inline auto ethernet_ipv4_udp_frame(std::uint16_t dst_port, const std::vector>& frames) -> std::vector { std::vector file; From 2cf6a2bf1d5422cd3bc24b9b0648a28361b51a63 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:01:48 +0100 Subject: [PATCH 091/144] ci: switch PyPI publish to Trusted Publishing (OIDC) Co-Authored-By: Claude Sonnet 5 --- .github/workflows/publish.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 550dbb6..c0c190a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -48,8 +48,12 @@ jobs: needs: [build_wheels, build_sdist] runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/v') + environment: + name: pypi + url: https://pypi.org/project/itchcpp/ permissions: - contents: write + contents: write + id-token: write steps: - name: Download all artifacts uses: actions/download-artifact@v4 @@ -61,7 +65,6 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist - password: ${{ secrets.ITCHCPP_PYPI_API_TOKEN }} - name: Create GitHub Release uses: softprops/action-gh-release@v2 From d092ea21520ff0806394c48f6cbd4f912aa44920 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:28 +0100 Subject: [PATCH 092/144] examples: add SoupBinTCP decode example Co-Authored-By: Claude Sonnet 5 --- examples/transport/soupbintcp_example.cpp | 133 ++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 examples/transport/soupbintcp_example.cpp diff --git a/examples/transport/soupbintcp_example.cpp b/examples/transport/soupbintcp_example.cpp new file mode 100644 index 0000000..c3dd006 --- /dev/null +++ b/examples/transport/soupbintcp_example.cpp @@ -0,0 +1,133 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/transport/soupbintcp.hpp" + +namespace { + +constexpr int BYTE_BITS = 8; +constexpr unsigned BYTE_MASK = 0xFF; +constexpr std::size_t SESSION_SIZE = 10; +constexpr std::size_t SEQUENCE_SIZE = 20; + +// Appends a complete SoupBinTCP packet (2-byte big-endian length, 1-byte type, +// payload) to the byte stream under construction. +auto append_packet( + std::vector& stream, + itch::transport::SoupBinPacketType type, + std::span payload +) -> void { + const auto length = static_cast(payload.size() + 1); + stream.push_back(static_cast((length >> BYTE_BITS) & BYTE_MASK)); + stream.push_back(static_cast(length & BYTE_MASK)); + stream.push_back(static_cast(type)); + stream.insert(stream.end(), payload.begin(), payload.end()); +} + +// Builds the Login Accepted payload: a space-padded session id followed by an +// ASCII, space-padded starting sequence number, per the SoupBinTCP spec. +auto make_login_accepted_payload(std::string_view session, std::uint64_t sequence) + -> std::vector { + std::string text {session}; + text.resize(SESSION_SIZE, ' '); + std::string sequence_text = std::format("{}", sequence); + sequence_text.resize(SEQUENCE_SIZE, ' '); + text += sequence_text; + + std::vector payload(text.size()); + for (std::size_t index = 0; index < text.size(); ++index) { + payload[index] = static_cast(text[index]); + } + return payload; +} + +} // namespace + +// Demonstrates decoding a SoupBinTCP byte stream directly: a Login Accepted +// control packet followed by sequenced ITCH data packets, with one packet +// deliberately split across two feed() calls to show the decoder reassembling +// a packet that TCP delivered across multiple reads. +auto main() -> int { + std::uint64_t decoded_count = 0; + itch::transport::SoupBinDecoder decoder {[&](const itch::Message& message) { + ++decoded_count; + const char type = std::visit([](const auto& msg) { return msg.message_type; }, message); + std::cout << std::format("decoded message {}: type '{}'\n", decoded_count, type); + }}; + decoder.set_event_callback([](itch::transport::SoupBinPacketType type, + std::span payload) { + std::cout << std::format( + "control packet: type '{}' ({} bytes)\n", static_cast(type), payload.size() + ); + }); + + const itch::Message add_order = itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200000000000ULL, + .order_reference_number = 1001, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1500000, + }; + const itch::Message executed = itch::OrderExecutedMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200500000000ULL, + .order_reference_number = 1001, + .executed_shares = 100, + .match_number = 5001, + }; + const itch::Message second_order = itch::AddOrderMessage { + .stock_locate = 2, + .tracking_number = 0, + .timestamp = 34201000000000ULL, + .order_reference_number = 1002, + .buy_sell_indicator = 'S', + .shares = 200, + .stock = {'M', 'S', 'F', 'T', ' ', ' ', ' ', ' '}, + .price = 3200000, + }; + + std::vector stream; + append_packet( + stream, + itch::transport::SoupBinPacketType::login_accepted, + make_login_accepted_payload("DEMO", 1) + ); + append_packet( + stream, itch::transport::SoupBinPacketType::sequenced_data, itch::encode_message(add_order) + ); + append_packet( + stream, itch::transport::SoupBinPacketType::sequenced_data, itch::encode_message(executed) + ); + append_packet( + stream, + itch::transport::SoupBinPacketType::sequenced_data, + itch::encode_message(second_order) + ); + + // Feed the stream in two chunks, splitting mid-packet, to prove feed() + // reassembles a packet spanning multiple TCP reads rather than requiring + // one call per packet. + const std::size_t split = stream.size() / 2; + decoder.feed(std::span {stream}.subspan(0, split)); + decoder.feed(std::span {stream}.subspan(split)); + + std::cout << std::format( + "session '{}', next sequence {}, {} messages decoded\n", + decoder.current_session(), + decoder.next_sequence(), + decoder.messages_decoded() + ); + return 0; +} From 925a515f4927da96e3ed4af459bea2c4f4a561ac Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:28 +0100 Subject: [PATCH 093/144] examples: add MoldUDP64 decode example Co-Authored-By: Claude Sonnet 5 --- examples/transport/moldudp64_example.cpp | 138 +++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 examples/transport/moldudp64_example.cpp diff --git a/examples/transport/moldudp64_example.cpp b/examples/transport/moldudp64_example.cpp new file mode 100644 index 0000000..15d9bce --- /dev/null +++ b/examples/transport/moldudp64_example.cpp @@ -0,0 +1,138 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/transport/moldudp64.hpp" +#include "itch/transport/sequencing.hpp" + +namespace { + +constexpr int BYTE_BITS = 8; +constexpr unsigned BYTE_MASK = 0xFF; +constexpr std::size_t SESSION_SIZE = 10; + +// Appends `width` bytes of `value` to `buffer`, most significant byte first. +auto append_big_endian(std::vector& buffer, std::uint64_t value, std::size_t width) + -> void { + for (std::size_t index = 0; index < width; ++index) { + const std::size_t shift = (width - 1 - index) * static_cast(BYTE_BITS); + buffer.push_back(static_cast((value >> shift) & BYTE_MASK)); + } +} + +// Builds one MoldUDP64 datagram: the 20-byte header followed by one +// length-prefixed message block (itch::encode_frame) per message. +auto make_packet( + std::string_view session, + std::uint64_t sequence_number, + const std::vector& messages +) -> std::vector { + std::string padded_session {session}; + padded_session.resize(SESSION_SIZE, ' '); + + std::vector packet; + for (const char letter : padded_session) { + packet.push_back(static_cast(letter)); + } + append_big_endian(packet, sequence_number, sizeof(std::uint64_t)); + append_big_endian(packet, messages.size(), sizeof(std::uint16_t)); + for (const auto& message : messages) { + const auto frame = itch::encode_frame(message); + packet.insert(packet.end(), frame.begin(), frame.end()); + } + return packet; +} + +// A minimal RetransmitRequester that only prints what it would ask for; a real +// implementation would issue a MoldUDP64 rewind/retransmit request here and +// feed the recovered messages back through decode_packet(). +class PrintingRetransmitRequester : public itch::transport::RetransmitRequester { + public: + auto request_retransmit( + std::string_view session, std::uint64_t start_sequence, std::uint64_t count + ) -> void override { + std::cout << std::format( + "retransmit request: session '{}', start {}, count {}\n", session, start_sequence, count + ); + } +}; + +} // namespace + +// Demonstrates decoding MoldUDP64 datagrams directly (no pcap capture +// involved): a normal packet followed by one that skips two sequence numbers, +// showing the embedded SequenceTracker detect the gap and drive a +// RetransmitRequester recovery hook. +auto main() -> int { + std::uint64_t decoded_count = 0; + itch::transport::MoldUdp64Decoder decoder {[&](const itch::Message& message) { + ++decoded_count; + const char type = std::visit([](const auto& msg) { return msg.message_type; }, message); + std::cout << std::format("decoded message {}: type '{}'\n", decoded_count, type); + }}; + + const itch::Message first_order = itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200000000000ULL, + .order_reference_number = 2001, + .buy_sell_indicator = 'B', + .shares = 300, + .stock = {'G', 'O', 'O', 'G', ' ', ' ', ' ', ' '}, + .price = 2750000, + }; + const itch::Message second_order = itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200100000000ULL, + .order_reference_number = 2002, + .buy_sell_indicator = 'S', + .shares = 150, + .stock = {'G', 'O', 'O', 'G', ' ', ' ', ' ', ' '}, + .price = 2751000, + }; + const itch::Message third_order = itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200200000000ULL, + .order_reference_number = 2003, + .buy_sell_indicator = 'B', + .shares = 75, + .stock = {'G', 'O', 'O', 'G', ' ', ' ', ' ', ' '}, + .price = 2749000, + }; + + const auto first_packet = make_packet("DEMO", 1, {first_order, second_order}); + decoder.decode_packet(first_packet); + + // Install the recovery hooks before feeding the gapped packet: sequence 1 + // carried 2 messages, so 3 is expected next, but the next packet starts at + // 5, leaving sequences 3 and 4 missing. + decoder.tracker().set_gap_callback( + [](std::string_view session, std::uint64_t expected, std::uint64_t received) { + std::cerr << std::format( + "gap on session '{}': expected {} but got {}\n", session, expected, received + ); + } + ); + PrintingRetransmitRequester requester; + decoder.tracker().set_retransmit_requester(&requester); + + const auto second_packet = make_packet("DEMO", 5, {third_order}); + decoder.decode_packet(second_packet); + + std::cout << std::format( + "{} packets decoded, {} messages decoded, {} gaps detected\n", + decoder.packets_decoded(), + decoder.messages_decoded(), + decoder.tracker().gap_count() + ); + return 0; +} From ad6f0970bc1b3d817b91a53a31f90f92e6f04979 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:28 +0100 Subject: [PATCH 094/144] examples: add zero-copy overlay example Co-Authored-By: Claude Sonnet 5 --- examples/overlay/overlay_example.cpp | 72 ++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 examples/overlay/overlay_example.cpp diff --git a/examples/overlay/overlay_example.cpp b/examples/overlay/overlay_example.cpp new file mode 100644 index 0000000..663fa8a --- /dev/null +++ b/examples/overlay/overlay_example.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/overlay.hpp" + +// Demonstrates itch::overlay::for_each_message: a zero-copy, lazy alternative +// to itch::Parser. The eager Parser decodes every field of every message up +// front; the overlay view only converts the specific fields the callback +// touches, which pays off when a caller only cares about a few fields. +auto main() -> int { + const itch::Message first_order = itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200000000000ULL, + .order_reference_number = 3001, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1500000, + }; + const itch::Message second_order = itch::AddOrderMessage { + .stock_locate = 2, + .tracking_number = 0, + .timestamp = 34200100000000ULL, + .order_reference_number = 3002, + .buy_sell_indicator = 'S', + .shares = 200, + .stock = {'M', 'S', 'F', 'T', ' ', ' ', ' ', ' '}, + .price = 3200000, + }; + const itch::Message third_order = itch::AddOrderMessage { + .stock_locate = 3, + .tracking_number = 0, + .timestamp = 34200200000000ULL, + .order_reference_number = 3003, + .buy_sell_indicator = 'B', + .shares = 50, + .stock = {'G', 'O', 'O', 'G', ' ', ' ', ' ', ' '}, + .price = 2750000, + }; + + std::vector buffer; + for (const auto& message : {first_order, second_order, third_order}) { + const auto frame = itch::encode_frame(message); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + + std::uint64_t viewed_count = 0; + const auto delivered = + itch::overlay::for_each_message(buffer, [&](const itch::overlay::MessageView& view) { + if (view.type() != 'A') { + return; + } + ++viewed_count; + // Only stock() and price() are decoded here; every other field of + // this Add Order frame (shares, side, order reference number, ...) + // is never converted from network byte order, unlike a Parser pass. + const itch::overlay::AddOrderView order {view.data(), view.size()}; + std::cout << std::format( + "order {}: {} @ {}\n", viewed_count, order.stock(), order.price() + ); + }); + + std::cout << std::format( + "{} of {} frames viewed as Add Order messages\n", viewed_count, delivered + ); + return 0; +} From d0661d2f04ae9cd7630b7bbb6bcd2f2bbee4d4d7 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:28 +0100 Subject: [PATCH 095/144] examples: add OHLCV bars example Co-Authored-By: Claude Sonnet 5 --- examples/analytics/bars_example.cpp | 109 ++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 examples/analytics/bars_example.cpp diff --git a/examples/analytics/bars_example.cpp b/examples/analytics/bars_example.cpp new file mode 100644 index 0000000..7b0cdd2 --- /dev/null +++ b/examples/analytics/bars_example.cpp @@ -0,0 +1,109 @@ +#include +#include +#include +#include +#include + +#include "itch/analytics/bars.hpp" +#include "itch/book/book_manager.hpp" +#include "itch/encoder.hpp" +#include "itch/parser.hpp" + +// Aggregates a synthetic trade tape into one-minute OHLCV bars with +// `BarBuilder` under a `TimeClock` policy, showing how the builder emits a +// completed bar each time the clock's bucket id advances. +auto main() -> int { + constexpr std::uint64_t base_timestamp {34'200'000'000'000ULL}; // 09:30:00.000 + constexpr std::uint64_t one_minute_ns {60'000'000'000ULL}; + + const std::vector trades { + itch::NonCrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp, + .order_reference_number = 1, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'500'000, + .match_number = 1 + }, + itch::NonCrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 10'000'000'000ULL, + .order_reference_number = 2, + .buy_sell_indicator = 'S', + .shares = 200, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'501'000, + .match_number = 2 + }, + itch::NonCrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + one_minute_ns + 10'000'000'000ULL, + .order_reference_number = 3, + .buy_sell_indicator = 'B', + .shares = 150, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'499'000, + .match_number = 3 + }, + itch::NonCrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + one_minute_ns + 30'000'000'000ULL, + .order_reference_number = 4, + .buy_sell_indicator = 'S', + .shares = 300, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'502'000, + .match_number = 4 + }, + itch::NonCrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + (2 * one_minute_ns) + 10'000'000'000ULL, + .order_reference_number = 5, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'503'000, + .match_number = 5 + }, + }; + + std::vector buffer; + for (const auto& trade : trades) { + const auto frame = itch::encode_frame(trade); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + + itch::analytics::BarBuilder bars { + itch::analytics::TimeClock {.interval_ns = one_minute_ns}, + [](const itch::analytics::Bar& bar) { + std::cout << std::format( + "Bar {}: open={:.4f} high={:.4f} low={:.4f} close={:.4f} volume={} trades={}\n", + bar.bucket, + bar.open, + bar.high, + bar.low, + bar.close, + bar.volume, + bar.trade_count + ); + } + }; + + itch::book::BookManager manager; + manager.set_trade_callback([&](const itch::Trade& trade) { bars.add(trade); }); + + itch::Parser parser; + parser.parse(std::span {buffer}, [&](const itch::Message& msg) { + manager.process(msg); + }); + bars.flush(); // Emit the final, partial bar. + + return 0; +} From c11f2e6a6e00c4b2b3f6f93ac29186c22d6531f1 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:29 +0100 Subject: [PATCH 096/144] examples: add microstructure example Co-Authored-By: Claude Sonnet 5 --- examples/analytics/microstructure_example.cpp | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 examples/analytics/microstructure_example.cpp diff --git a/examples/analytics/microstructure_example.cpp b/examples/analytics/microstructure_example.cpp new file mode 100644 index 0000000..372bd12 --- /dev/null +++ b/examples/analytics/microstructure_example.cpp @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include + +#include "itch/analytics/microstructure.hpp" +#include "itch/book/book_manager.hpp" +#include "itch/encoder.hpp" +#include "itch/parser.hpp" + +// Computes spread, mid price, queue imbalance, depth, and order-flow +// imbalance from a live `L3Book`, built from a small in-code sequence of +// resting orders so the example runs standalone without an external ITCH file. +auto main() -> int { + constexpr std::uint64_t base_timestamp {34'200'000'000'000ULL}; // 09:30:00.000 + + const std::vector orders { + itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp, + .order_reference_number = 1, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'499'000 + }, + itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 100, + .order_reference_number = 2, + .buy_sell_indicator = 'B', + .shares = 200, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'498'000 + }, + itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 200, + .order_reference_number = 3, + .buy_sell_indicator = 'S', + .shares = 150, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'501'000 + }, + itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 300, + .order_reference_number = 4, + .buy_sell_indicator = 'S', + .shares = 250, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'502'000 + }, + itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 400, + .order_reference_number = 5, + .buy_sell_indicator = 'B', + .shares = 300, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'500'000 // Improves the best bid. + }, + }; + + std::vector buffer; + for (const auto& order : orders) { + const auto frame = itch::encode_frame(order); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + + itch::book::BookManager manager; + manager.track_symbol("AAPL"); + + std::vector snapshots; + manager.set_bbo_callback([&](const itch::book::L3Book&, const itch::book::Bbo& bbo) { + snapshots.push_back(bbo); + }); + + itch::Parser parser; + parser.parse(std::span {buffer}, [&](const itch::Message& msg) { + manager.process(msg); + }); + + const itch::book::L3Book* book = manager.book_for_symbol("AAPL"); + if (book == nullptr) { + std::cerr << "Error: book for AAPL was not created.\n"; + return 1; + } + + const itch::book::Bbo current = book->bbo(); + std::cout << std::format( + "Spread: {:.4f} Mid: {:.4f} Queue imbalance: {:.4f}\n", + itch::analytics::spread(current), + itch::analytics::mid_price(current), + itch::analytics::queue_imbalance(current) + ); + + const auto bid_depth = itch::analytics::depth_at_level(*book, itch::book::Side::buy, 2); + const auto ask_depth = itch::analytics::depth_at_level(*book, itch::book::Side::sell, 2); + std::cout << std::format("Depth (2 levels): bid={} ask={}\n", bid_depth, ask_depth); + + if (snapshots.size() >= 2) { + const double flow_imbalance = itch::analytics::order_flow_imbalance( + snapshots[snapshots.size() - 2], snapshots.back() + ); + std::cout << std::format( + "Order-flow imbalance (last BBO change): {:.2f}\n", flow_imbalance + ); + } + + return 0; +} From c8118033a08d64c296920e00e0a200179b5e0165 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:29 +0100 Subject: [PATCH 097/144] examples: add auction/imbalance example Co-Authored-By: Claude Sonnet 5 --- examples/analytics/auctions_example.cpp | 96 +++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 examples/analytics/auctions_example.cpp diff --git a/examples/analytics/auctions_example.cpp b/examples/analytics/auctions_example.cpp new file mode 100644 index 0000000..370fbe4 --- /dev/null +++ b/examples/analytics/auctions_example.cpp @@ -0,0 +1,96 @@ +#include +#include +#include +#include +#include + +#include "itch/analytics/auctions.hpp" +#include "itch/analytics/imbalance.hpp" +#include "itch/encoder.hpp" +#include "itch/parser.hpp" + +// Reconstructs an opening auction from the NOII imbalance messages leading up +// to a cross with `AuctionTracker`, alongside decoding a standalone NOII +// message directly with `make_imbalance_info`. +auto main() -> int { + constexpr std::uint64_t base_timestamp {34'200'000'000'000ULL}; // 09:30:00.000 + + const itch::NOIIMessage first_noii { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp, + .paired_shares = 50'000, + .imbalance_shares = 5'000, + .imbalance_direction = 'B', + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .far_price = 1'499'500, + .near_price = 1'500'000, + .current_reference_price = 1'500'000, + .cross_type = 'O', + .price_variation_indicator = 'L' + }; + + // Demonstrate decoding a single NOII message directly. + const itch::analytics::ImbalanceInfo decoded = itch::analytics::make_imbalance_info(first_noii); + std::cout << std::format( + "NOII for {}: {} shares imbalance ({}), reference price {:.4f}\n", + decoded.stock, + decoded.imbalance_shares, + itch::analytics::imbalance_direction_name(decoded.imbalance_direction), + decoded.current_reference_price + ); + + const std::vector stream { + itch::Message {first_noii}, + itch::Message {itch::NOIIMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 1'000'000'000ULL, + .paired_shares = 60'000, + .imbalance_shares = 2'000, + .imbalance_direction = 'B', + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .far_price = 1'500'000, + .near_price = 1'500'000, + .current_reference_price = 1'500'000, + .cross_type = 'O', + .price_variation_indicator = 'L' + }}, + itch::Message {itch::CrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 2'000'000'000ULL, + .shares = 62'000, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .cross_price = 1'500'000, + .match_number = 1, + .cross_type = 'O' + }}, + }; + + std::vector buffer; + for (const auto& msg : stream) { + const auto frame = itch::encode_frame(msg); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + + itch::analytics::AuctionTracker tracker; + tracker.set_auction_callback([](const itch::analytics::Auction& auction) { + std::cout << std::format( + "{} for {}: {} shares @ {:.4f} (paired {}, imbalance {})\n", + itch::analytics::cross_type_name(auction.cross_type), + auction.stock, + auction.cross_shares, + auction.cross_price, + auction.paired_shares, + auction.imbalance_shares + ); + }); + + itch::Parser parser; + parser.parse(std::span {buffer}, [&](const itch::Message& msg) { + tracker.process(msg); + }); + + return 0; +} From bd6b2d947ca9a68ac7b849546b805b6169d06b88 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:29 +0100 Subject: [PATCH 098/144] examples: add CSV sink example Co-Authored-By: Claude Sonnet 5 --- examples/io/csv_sink_example.cpp | 89 ++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 examples/io/csv_sink_example.cpp diff --git a/examples/io/csv_sink_example.cpp b/examples/io/csv_sink_example.cpp new file mode 100644 index 0000000..df54c5f --- /dev/null +++ b/examples/io/csv_sink_example.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/io/csv_sink.hpp" +#include "itch/messages.hpp" + +namespace { + +// ITCH text fields are fixed-width and space-padded on the wire, not +// null-terminated C strings, so a plain string-literal designated initializer +// cannot fill them (there is no room for the implicit '\0'). This copies +// `value` in and pads the remainder with spaces instead. +template +auto fill_field(char (&dest)[Size], std::string_view value) -> void { + std::ranges::fill(dest, ' '); + std::ranges::copy(value, dest); +} + +auto make_sample_messages() -> std::vector { + const itch::SystemEventMessage system_event { + .stock_locate = 0, + .tracking_number = 1, + .timestamp = 34'200'000'000'000ULL, + .event_code = 'O', + }; + + itch::AddOrderMessage add_order { + .stock_locate = 1, + .tracking_number = 1, + .timestamp = 34'200'000'500'000ULL, + .order_reference_number = 1001, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {}, + .price = 1500000, + }; + fill_field(add_order.stock, "AAPL"); + + itch::StockTradingActionMessage trading_action { + .stock_locate = 1, + .tracking_number = 1, + .timestamp = 34'200'001'000'000ULL, + .stock = {}, + .trading_state = 'T', + .reserved = ' ', + .reason = {}, + }; + fill_field(trading_action.stock, "AAPL"); + fill_field(trading_action.reason, "T"); + + return {system_event, add_order, trading_action}; +} + +} // namespace + +// Demonstrates CsvSink as a concrete itch::io::MessageSink: every one of the 23 +// heterogeneous ITCH message structs flattens into the same wide CSV row schema, +// so downstream tooling never has to special-case which type it received. +auto main() -> int { + const std::vector messages = make_sample_messages(); + + const auto csv_path = std::filesystem::temp_directory_path() / "itchcpp_csv_sink_example.csv"; + std::ofstream out_file {csv_path}; + + itch::io::CsvSink sink {out_file}; + itch::io::MessageSink& generic_sink = sink; // exercise through the abstract interface + for (const auto& message : messages) { + generic_sink.write(message); + } + generic_sink.flush(); + out_file.close(); + + std::cout << std::format("Wrote {} CSV rows to {}\n", sink.rows_written(), csv_path.string()); + + std::ifstream in_file {csv_path}; + std::string line; + while (std::getline(in_file, line)) { + std::cout << line << '\n'; + } + return 0; +} From 83a6022f660824cb853298a7e0648732642dd700 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:29 +0100 Subject: [PATCH 099/144] examples: add Arrow/Parquet export example Co-Authored-By: Claude Sonnet 5 --- examples/io/arrow_export_example.cpp | 88 ++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 examples/io/arrow_export_example.cpp diff --git a/examples/io/arrow_export_example.cpp b/examples/io/arrow_export_example.cpp new file mode 100644 index 0000000..0f06c85 --- /dev/null +++ b/examples/io/arrow_export_example.cpp @@ -0,0 +1,88 @@ +#include +#include + +#ifdef ITCH_WITH_ARROW +#include +#include +#include +#include +#include + +#include "itch/io/arrow_export.hpp" +#include "itch/messages.hpp" + +namespace { + +// ITCH text fields are fixed-width and space-padded on the wire, not +// null-terminated C strings, so a plain string-literal designated initializer +// cannot fill them (there is no room for the implicit '\0'). This copies +// `value` in and pads the remainder with spaces instead. +template +auto fill_field(char (&dest)[Size], std::string_view value) -> void { + std::ranges::fill(dest, ' '); + std::ranges::copy(value, dest); +} + +} // namespace +#endif // ITCH_WITH_ARROW + +// Demonstrates ArrowExporter turning a handful of synthesized messages into a +// Parquet file, the columnar hand-off point for pandas/Polars/DuckDB/Spark +// pipelines. Only meaningful when the library is built with +// -DITCH_WITH_ARROW=ON; otherwise this file still compiles but does nothing. +auto main() -> int { +#ifdef ITCH_WITH_ARROW + const itch::SystemEventMessage system_event { + .stock_locate = 0, + .tracking_number = 1, + .timestamp = 34'200'000'000'000ULL, + .event_code = 'O', + }; + + itch::AddOrderMessage buy_order { + .stock_locate = 1, + .tracking_number = 1, + .timestamp = 34'200'000'500'000ULL, + .order_reference_number = 1001, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {}, + .price = 1500000, + }; + fill_field(buy_order.stock, "AAPL"); + + itch::AddOrderMessage sell_order { + .stock_locate = 1, + .tracking_number = 1, + .timestamp = 34'200'001'000'000ULL, + .order_reference_number = 1002, + .buy_sell_indicator = 'S', + .shares = 200, + .stock = {}, + .price = 1500500, + }; + fill_field(sell_order.stock, "AAPL"); + + const std::vector messages {system_event, buy_order, sell_order}; + + itch::io::ArrowExporter exporter; + for (const auto& message : messages) { + exporter.append(message); + } + + const auto parquet_path = + std::filesystem::temp_directory_path() / "itchcpp_arrow_export_example.parquet"; + const bool wrote_ok = exporter.write_parquet(parquet_path.string()); + if (!wrote_ok) { + std::cerr << std::format("Failed to write Parquet file: {}\n", exporter.error()); + return 1; + } + + std::cout << std::format( + "Appended {} rows and wrote Parquet file to {}\n", exporter.rows(), parquet_path.string() + ); +#else + std::cout << "This example requires -DITCH_WITH_ARROW=ON\n"; +#endif // ITCH_WITH_ARROW + return 0; +} From 2d29fb0fe444cb9dfb8a182f1173e402444a0967 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:30 +0100 Subject: [PATCH 100/144] examples: add encoder round-trip example Co-Authored-By: Claude Sonnet 5 --- examples/encoder/encoder_example.cpp | 77 ++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 examples/encoder/encoder_example.cpp diff --git a/examples/encoder/encoder_example.cpp b/examples/encoder/encoder_example.cpp new file mode 100644 index 0000000..e57e77f --- /dev/null +++ b/examples/encoder/encoder_example.cpp @@ -0,0 +1,77 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/parser.hpp" + +namespace { + +// ITCH text fields are fixed-width and space-padded on the wire, not +// null-terminated C strings, so a plain string-literal designated initializer +// cannot fill them (there is no room for the implicit '\0'). This copies +// `value` in and pads the remainder with spaces instead. +template +auto fill_field(char (&dest)[Size], std::string_view value) -> void { + std::ranges::fill(dest, ' '); + std::ranges::copy(value, dest); +} + +auto make_sample_order() -> itch::AddOrderMessage { + itch::AddOrderMessage order { + .stock_locate = 7, + .tracking_number = 3, + .timestamp = 34'200'000'123'456ULL, + .order_reference_number = 987654321, + .buy_sell_indicator = 'B', + .shares = 500, + .stock = {}, + .price = 1500000, + }; + fill_field(order.stock, "AAPL"); + return order; +} + +} // namespace + +// Demonstrates the encoder as the inverse of the parser: it serializes a message +// back to wire bytes and guarantees parse(encode(msg)) == msg, which is what +// lets tests and scenario generators synthesize valid ITCH streams instead of +// depending on captured fixture files. +auto main() -> int { + const itch::AddOrderMessage original = make_sample_order(); + const itch::Message message = original; + + const std::vector body = itch::encode_message(message); + const std::vector frame = itch::encode_frame(message); + std::cout << std::format( + "Encoded body: {} bytes; framed (2-byte length prefix): {} bytes\n", + body.size(), + frame.size() + ); + + itch::Parser parser; + const std::vector decoded_messages = + parser.parse(std::span {frame}); + std::cout << std::format( + "Parser decoded {} message(s) from the frame\n", decoded_messages.size() + ); + + // Message has no operator== of its own, so the round-trip guarantee is + // verified field by field on the decoded alternative instead. + const auto& decoded = std::get(decoded_messages.front()); + const bool round_trips = decoded.order_reference_number == original.order_reference_number && + decoded.price == original.price && decoded.shares == original.shares && + decoded.buy_sell_indicator == original.buy_sell_indicator && + decoded.timestamp == original.timestamp; + std::cout << std::format( + "Round-trip parse(encode(msg)) == msg: {}\n", round_trips ? "holds" : "FAILED" + ); + return 0; +} From 1347cf388f3f0747bd14c1bbc42dbff82dcb735a Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:30 +0100 Subject: [PATCH 101/144] examples: add replay engine example Co-Authored-By: Claude Sonnet 5 --- examples/replay/replay_example.cpp | 84 ++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 examples/replay/replay_example.cpp diff --git a/examples/replay/replay_example.cpp b/examples/replay/replay_example.cpp new file mode 100644 index 0000000..bbd347a --- /dev/null +++ b/examples/replay/replay_example.cpp @@ -0,0 +1,84 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/replay.hpp" + +namespace { + +// ITCH text fields are fixed-width and space-padded on the wire, not +// null-terminated C strings, so a plain string-literal designated initializer +// cannot fill them (there is no room for the implicit '\0'). This copies +// `value` in and pads the remainder with spaces instead. +template +auto fill_field(char (&dest)[Size], std::string_view value) -> void { + std::ranges::fill(dest, ' '); + std::ranges::copy(value, dest); +} + +auto make_order(std::uint32_t index, std::uint64_t session_start_ns, std::uint64_t message_gap_ns) + -> itch::AddOrderMessage { + itch::AddOrderMessage order { + .stock_locate = 1, + .tracking_number = 1, + .timestamp = session_start_ns + static_cast(index) * message_gap_ns, + .order_reference_number = 1000ULL + static_cast(index), + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {}, + .price = 1'500'000U + (index * 100U), + }; + fill_field(order.stock, "AAPL"); + return order; +} + +} // namespace + +// Demonstrates ReplayEngine pacing a synthesized feed by its own timestamps +// rather than delivering it as fast as the CPU can parse, which is what makes +// it useful for realistic backtesting and system simulation instead of a plain +// Parser::parse loop. +auto main() -> int { + constexpr std::uint64_t session_start_ns = 34'200'000'000'000ULL; // 09:30:00.000000000 + constexpr std::uint64_t message_gap_ns = 100'000'000ULL; // 100 ms apart in feed time + constexpr std::uint32_t order_count = 4; + + std::vector stream; + for (std::uint32_t index = 0; index < order_count; ++index) { + const itch::AddOrderMessage order = make_order(index, session_start_ns, message_gap_ns); + const std::vector frame = itch::encode_frame(itch::Message {order}); + stream.insert(stream.end(), frame.begin(), frame.end()); + } + + // At 200x speed, the 100 ms feed-time gaps between messages become ~0.5 ms + // wall-clock sleeps, so the example finishes almost instantly while still + // exercising real pacing rather than a no-op. + itch::ReplayEngine engine {200.0}; + std::uint64_t messages_seen = 0; + const auto start = std::chrono::steady_clock::now(); + const std::uint64_t replayed = + engine.replay(std::span {stream}, [&](const itch::Message& message) { + ++messages_seen; + const auto& order = std::get(message); + std::cout << std::format( + "Replayed order #{} at {} ns past midnight\n", messages_seen, order.timestamp + ); + }); + const auto elapsed = std::chrono::steady_clock::now() - start; + + std::cout << std::format( + "Replayed {} messages in {} ms wall-clock at {}x speed\n", + replayed, + std::chrono::duration_cast(elapsed).count(), + engine.speed() + ); + return 0; +} From b9c18ee8d8d9e03e7ebd798e0844925081c77eef Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:30 +0100 Subject: [PATCH 102/144] examples: add price/time/indicators example Co-Authored-By: Claude Sonnet 5 --- .../price_time_indicators_example.cpp | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 examples/utilities/price_time_indicators_example.cpp diff --git a/examples/utilities/price_time_indicators_example.cpp b/examples/utilities/price_time_indicators_example.cpp new file mode 100644 index 0000000..24aad9b --- /dev/null +++ b/examples/utilities/price_time_indicators_example.cpp @@ -0,0 +1,53 @@ +#include +#include +#include +#include + +#include "itch/indicators.hpp" +#include "itch/price.hpp" +#include "itch/time.hpp" + +// The smallest, most beginner-friendly example in this directory: shows the +// three tiny utility types that keep raw ITCH wire values (integer prices, +// nanosecond timestamps, single-letter codes) from turning into silent +// off-by-a-decimal or look-the-code-up-yourself bugs in application code. +auto main() -> int { + // A raw ITCH price is just an integer with an implied decimal point: the + // same integer means different things depending on the field's scale, so + // dividing by 10000 (or 100000000) by hand everywhere risks picking the + // wrong divisor. StandardPrice/MwcbPrice bake the scale into the type. + const itch::StandardPrice trade_price = itch::make_price(1500000); // 4 implied decimals + const itch::MwcbPrice level1_price = + itch::make_mwcb_price(410250000000ULL); // 8 implied decimals + std::cout << std::format( + "Trade price raw {} -> {}\n", trade_price.raw(), trade_price.to_string() + ); + std::cout << std::format( + "MWCB level 1 raw {} -> {}\n", level1_price.raw(), level1_price.to_string() + ); + + // ITCH timestamps are raw nanoseconds since midnight with no date attached. + // These helpers turn that offset into readable clock time, or a full + // calendar time point given a session date, without hand-rolled div/mod math. + constexpr std::uint64_t market_open_ns = 34'200'000'000'000ULL; // 09:30:00.000000000 + std::cout << std::format( + "Market open time of day: {}\n", itch::format_timestamp(market_open_ns) + ); + const std::chrono::year_month_day session_date { + std::chrono::year {2026}, std::chrono::month {7}, std::chrono::day {10} + }; + std::cout << std::format( + "Market open full timestamp: {}\n", itch::format_time_point(session_date, market_open_ns) + ); + + // Single-letter wire codes (event codes, trading states, ...) are meaningless + // without a lookup table; indicators.hpp supplies compile-time tables instead + // of scattering string literals through every call site that needs one. + std::cout << std::format( + "Event code 'O' means: {}\n", itch::indicators::SYSTEM_EVENT_CODES.at_or('O', "Unknown") + ); + std::cout << std::format( + "Trading state 'T' means: {}\n", itch::indicators::TRADING_STATES.at_or('T', "Unknown") + ); + return 0; +} From fd54312175c57ece088f2a48248f5d459f1e181c Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:30 +0100 Subject: [PATCH 103/144] examples: add Twap section to vwap example Co-Authored-By: Claude Sonnet 5 --- examples/analytics/vwap_example.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/analytics/vwap_example.cpp b/examples/analytics/vwap_example.cpp index 3ff3877..0c7f4fd 100644 --- a/examples/analytics/vwap_example.cpp +++ b/examples/analytics/vwap_example.cpp @@ -7,8 +7,8 @@ #include "itch/book/book_manager.hpp" #include "itch/parser.hpp" -// Computes the session VWAP for a chosen symbol by driving the BookManager's -// trade tape through a running Vwap accumulator. +// Computes the session VWAP and TWAP for a chosen symbol by driving the +// BookManager's trade tape through running Vwap and Twap accumulators. auto main(int argc, char* argv[]) -> int { if (argc < 3) { std::cerr << std::format("Usage: {} \n", argv[0]); @@ -26,9 +26,11 @@ auto main(int argc, char* argv[]) -> int { manager.track_symbol(argv[2]); itch::analytics::Vwap vwap; + itch::analytics::Twap twap; manager.set_trade_callback([&](const itch::Trade& trade) { if (trade.symbol == argv[2]) { vwap.add(trade.price, trade.shares); + twap.add(trade.price, trade.timestamp); } }); @@ -40,5 +42,9 @@ auto main(int argc, char* argv[]) -> int { std::cout << std::format( "VWAP for {}: {:.4f} over {} shares\n", argv[2], vwap.value(), vwap.volume() ); + + // Twap integrates the prevailing trade price over elapsed time rather than + // volume, so it is reported separately from the volume-weighted figure above. + std::cout << std::format("TWAP for {}: {:.4f}\n", argv[2], twap.value()); return 0; } From 153651944dab720951b0fec4da3bcd48c51cd17f Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:02:30 +0100 Subject: [PATCH 104/144] build: wire up new example targets Co-Authored-By: Claude Sonnet 5 --- examples/CMakeLists.txt | 90 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index cafc608..45e02b5 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -39,3 +39,93 @@ target_link_libraries( itch::itch warnings::strict ) + +add_executable(soupbintcp_example transport/soupbintcp_example.cpp) + +target_link_libraries( + soupbintcp_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(moldudp64_example transport/moldudp64_example.cpp) + +target_link_libraries( + moldudp64_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(overlay_example overlay/overlay_example.cpp) + +target_link_libraries( + overlay_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(bars_example analytics/bars_example.cpp) + +target_link_libraries( + bars_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(microstructure_example analytics/microstructure_example.cpp) + +target_link_libraries( + microstructure_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(auctions_example analytics/auctions_example.cpp) + +target_link_libraries( + auctions_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(csv_sink_example io/csv_sink_example.cpp) + +target_link_libraries( + csv_sink_example PRIVATE + itch::itch + warnings::strict +) + +if(ITCH_WITH_ARROW) + add_executable(arrow_export_example io/arrow_export_example.cpp) + + target_link_libraries( + arrow_export_example PRIVATE + itch::itch + warnings::strict + ) +endif() + +add_executable(encoder_example encoder/encoder_example.cpp) + +target_link_libraries( + encoder_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(replay_example replay/replay_example.cpp) + +target_link_libraries( + replay_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(price_time_indicators_example utilities/price_time_indicators_example.cpp) + +target_link_libraries( + price_time_indicators_example PRIVATE + itch::itch + warnings::strict +) From 883102906cb1c3aab6c4ad370d3aae9a8c2c96b0 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:14:11 +0100 Subject: [PATCH 105/144] fix: restore root vcpkg.json, required by CI's run-vcpkg baseline Co-Authored-By: Claude Sonnet 5 --- vcpkg.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 vcpkg.json diff --git a/vcpkg.json b/vcpkg.json new file mode 100644 index 0000000..f268b88 --- /dev/null +++ b/vcpkg.json @@ -0,0 +1,26 @@ +{ + "dependencies": [ + "gtest", + "benchmark" + ], + "features": { + "python": { + "description": "Build the pybind11 Python bindings", + "dependencies": [ + "pybind11" + ] + }, + "arrow": { + "description": "Enable Apache Arrow / Parquet columnar export", + "dependencies": [ + { + "name": "arrow", + "features": [ + "parquet" + ] + } + ] + } + }, + "builtin-baseline": "5bf0c55239da398b8c6f450818c9e28d36bf9966" +} From 7fea64ad147a58f1ffb0cce4bdf9bf95dc689655 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:16:08 +0100 Subject: [PATCH 106/144] updates --- packaging | 1 + vcpkg.json | 19 ------------------- 2 files changed, 1 insertion(+), 19 deletions(-) create mode 160000 packaging diff --git a/packaging b/packaging new file mode 160000 index 0000000..115a19b --- /dev/null +++ b/packaging @@ -0,0 +1 @@ +Subproject commit 115a19b0306f80635bf2911e3b1d4a4ab409cfbf diff --git a/vcpkg.json b/vcpkg.json index f268b88..165ef1b 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -3,24 +3,5 @@ "gtest", "benchmark" ], - "features": { - "python": { - "description": "Build the pybind11 Python bindings", - "dependencies": [ - "pybind11" - ] - }, - "arrow": { - "description": "Enable Apache Arrow / Parquet columnar export", - "dependencies": [ - { - "name": "arrow", - "features": [ - "parquet" - ] - } - ] - } - }, "builtin-baseline": "5bf0c55239da398b8c6f450818c9e28d36bf9966" } From f92f998ba34d468243c2beb70d1791cf6611f284 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:30:35 +0100 Subject: [PATCH 107/144] fix: include before checking __cpp_lib_expected Co-Authored-By: Claude Sonnet 5 --- include/itch/parser.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/itch/parser.hpp b/include/itch/parser.hpp index 6cb3e2b..817aac8 100644 --- a/include/itch/parser.hpp +++ b/include/itch/parser.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #ifdef __cpp_lib_expected From 7d43c19fab18c7f963bd306c1404a74a2730f00d Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:31 +0100 Subject: [PATCH 108/144] ci: pin format-check to clang-format 18.1.8 Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7baca6f..03a2aec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -88,7 +88,7 @@ jobs: uses: actions/checkout@v4 - name: Install clang-format - run: sudo apt-get update && sudo apt-get install -y clang-format + run: pip install clang-format==18.1.8 - name: Check formatting run: | From f60f03c930a58be3800faa342bd367872e7e4e45 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:32 +0100 Subject: [PATCH 109/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- examples/encoder/encoder_example.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/encoder/encoder_example.cpp b/examples/encoder/encoder_example.cpp index e57e77f..ddadf87 100644 --- a/examples/encoder/encoder_example.cpp +++ b/examples/encoder/encoder_example.cpp @@ -65,8 +65,8 @@ auto main() -> int { // Message has no operator== of its own, so the round-trip guarantee is // verified field by field on the decoded alternative instead. - const auto& decoded = std::get(decoded_messages.front()); - const bool round_trips = decoded.order_reference_number == original.order_reference_number && + const auto& decoded = std::get(decoded_messages.front()); + const bool round_trips = decoded.order_reference_number == original.order_reference_number && decoded.price == original.price && decoded.shares == original.shares && decoded.buy_sell_indicator == original.buy_sell_indicator && decoded.timestamp == original.timestamp; From 6099c601ef474f4a9eacd4f5892d3b80d80bcd56 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:32 +0100 Subject: [PATCH 110/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- include/itch/analytics/bars.hpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/include/itch/analytics/bars.hpp b/include/itch/analytics/bars.hpp index 2ef941f..df9a012 100644 --- a/include/itch/analytics/bars.hpp +++ b/include/itch/analytics/bars.hpp @@ -37,7 +37,7 @@ using BarCallback = std::function; /// @brief A clock that buckets trades by wall-clock time (nanoseconds). struct TimeClock { - std::uint64_t interval_ns {1}; + std::uint64_t interval_ns {1}; /// @brief Computes the bucket id for `trade` from its wall-clock timestamp. /// @@ -51,7 +51,7 @@ struct TimeClock { /// @brief A clock that buckets a fixed number of trades into each bar. struct TickClock { - std::uint64_t ticks_per_bar {1}; + std::uint64_t ticks_per_bar {1}; /// @brief Computes the bucket id for the current trade from its tick position. /// @@ -65,15 +65,14 @@ struct TickClock { /// @brief A clock that buckets a fixed amount of traded volume into each bar. struct VolumeClock { - std::uint64_t volume_per_bar {1}; + std::uint64_t volume_per_bar {1}; /// @brief Computes the bucket id for the current trade from cumulative volume. /// /// @param cumulative_volume Total shares traded so far, including this trade. /// @return The bucket id, `cumulative_volume / volume_per_bar`. - [[nodiscard]] auto bucket( - const Trade&, std::uint64_t cumulative_volume, std::uint64_t - ) const noexcept -> std::uint64_t { + [[nodiscard]] auto bucket(const Trade&, std::uint64_t cumulative_volume, std::uint64_t) + const noexcept -> std::uint64_t { return cumulative_volume / volume_per_bar; } }; From 9680caa7c1c77b4cd3826d228de15e02f3b01e63 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:32 +0100 Subject: [PATCH 111/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- include/itch/book/l3_book.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/itch/book/l3_book.hpp b/include/itch/book/l3_book.hpp index 8e9776e..ccf3c90 100644 --- a/include/itch/book/l3_book.hpp +++ b/include/itch/book/l3_book.hpp @@ -142,8 +142,8 @@ class L3Book { /// @param reference_number Exchange order reference number to look up. /// @return The order's raw limit price, or `std::nullopt` if no such /// order is resting. - [[nodiscard]] auto order_price(std::uint64_t reference_number) const - -> std::optional; + [[nodiscard]] auto order_price(std::uint64_t reference_number + ) const -> std::optional; /// @brief The side of a resting order, if present. /// @param reference_number Exchange order reference number to look up. From 72d71805a6d109df0eaf00a1edc6b62621bd2bc2 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:32 +0100 Subject: [PATCH 112/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- include/itch/indicators.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/itch/indicators.hpp b/include/itch/indicators.hpp index 4060db7..3d17758 100644 --- a/include/itch/indicators.hpp +++ b/include/itch/indicators.hpp @@ -50,8 +50,8 @@ class FrozenMap { /// /// @param key The key to look up. /// @return The matching description, or `std::nullopt` if no entry has this key. - [[nodiscard]] constexpr auto find(const KeyType& key) const noexcept - -> std::optional { + [[nodiscard]] constexpr auto find(const KeyType& key + ) const noexcept -> std::optional { const auto iter = std::ranges::lower_bound(m_entries, key, {}, &value_type::first); if (iter != m_entries.end() && iter->first == key) { return iter->second; From 4787d102cb52292c3d0b096a47d526172451abbc Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:32 +0100 Subject: [PATCH 113/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- include/itch/io/arrow_export.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/itch/io/arrow_export.hpp b/include/itch/io/arrow_export.hpp index 78cc5f8..3082f41 100644 --- a/include/itch/io/arrow_export.hpp +++ b/include/itch/io/arrow_export.hpp @@ -40,15 +40,15 @@ class ArrowExporter { ~ArrowExporter(); /// @brief Non-copyable; move-only. - ArrowExporter(const ArrowExporter&) = delete; + ArrowExporter(const ArrowExporter&) = delete; /// @brief Non-copyable; move-only. /// /// @return Reference to this exporter. - auto operator=(const ArrowExporter&) -> ArrowExporter& = delete; + auto operator=(const ArrowExporter&) -> ArrowExporter& = delete; /// @brief Move-constructs an exporter, transferring ownership of its state. - ArrowExporter(ArrowExporter&&) noexcept = default; + ArrowExporter(ArrowExporter&&) noexcept = default; /// @brief Move-assigns an exporter, transferring ownership of its state. /// From d9df73c5e1f28d90a30cd1508641d8d46df7e2ce Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:32 +0100 Subject: [PATCH 114/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- include/itch/io/sink.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/itch/io/sink.hpp b/include/itch/io/sink.hpp index 7656df0..efa9125 100644 --- a/include/itch/io/sink.hpp +++ b/include/itch/io/sink.hpp @@ -21,18 +21,18 @@ namespace itch::io { class MessageSink { public: /// @brief Default-constructs an empty sink. - MessageSink() = default; + MessageSink() = default; /// @brief Copy-constructs a sink. - MessageSink(const MessageSink&) = default; + MessageSink(const MessageSink&) = default; /// @brief Move-constructs a sink. - MessageSink(MessageSink&&) noexcept = default; + MessageSink(MessageSink&&) noexcept = default; /// @brief Copy-assigns a sink. /// /// @return Reference to this sink. - auto operator=(const MessageSink&) -> MessageSink& = default; + auto operator=(const MessageSink&) -> MessageSink& = default; /// @brief Move-assigns a sink. /// @@ -40,7 +40,7 @@ class MessageSink { auto operator=(MessageSink&&) noexcept -> MessageSink& = default; /// @brief Virtual destructor for safe polymorphic destruction. - virtual ~MessageSink() = default; + virtual ~MessageSink() = default; /// @brief Consumes one parsed message. /// From 1c14518175c2ba341353d28705b4af78fa557e66 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:33 +0100 Subject: [PATCH 115/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- include/itch/messages.hpp | 265 +++++++++++++++++++------------------- 1 file changed, 133 insertions(+), 132 deletions(-) diff --git a/include/itch/messages.hpp b/include/itch/messages.hpp index 1f8a24a..ec0d92c 100644 --- a/include/itch/messages.hpp +++ b/include/itch/messages.hpp @@ -47,24 +47,24 @@ struct SystemEventMessage { /// and displayed correctly. A security not named in any Stock Directory message /// should not appear in later messages for that day. struct StockDirectoryMessage { - char message_type = 'R'; ///< Always 'R'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - char stock[8]; ///< Stock symbol, right padded with spaces. - char market_category; ///< Listing market (see indicators::MARKET_CATEGORY). - char financial_status_indicator; ///< Financial status of the issuer. - uint32_t round_lot_size; ///< Number of shares in a round lot. - char round_lots_only; ///< 'Y' if only round lots may be entered, else 'N'. - char issue_classification; ///< Security class (see indicators::ISSUE_CLASSIFICATION_VALUES). - char issue_sub_type[2]; ///< Security sub-type (see indicators::ISSUE_SUB_TYPE_VALUES). - char authenticity; ///< 'P' production / 'T' test security. - char short_sale_threshold_indicator; ///< Reg SHO threshold: 'Y','N',' '. - char ipo_flag; ///< 'Y' if a new IPO, 'N' if not, ' ' if not available. - char luld_ref; ///< LULD reference price tier. - char etp_flag; ///< 'Y' if an exchange-traded product, else 'N'. - uint32_t etp_leverage_factor; ///< Leverage factor of the ETP (if applicable). - char inverse_indicator; ///< 'Y' if the ETP is an inverse product. + char message_type = 'R'; ///< Always 'R'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char market_category; ///< Listing market (see indicators::MARKET_CATEGORY). + char financial_status_indicator; ///< Financial status of the issuer. + uint32_t round_lot_size; ///< Number of shares in a round lot. + char round_lots_only; ///< 'Y' if only round lots may be entered, else 'N'. + char issue_classification; ///< Security class (see indicators::ISSUE_CLASSIFICATION_VALUES). + char issue_sub_type[2]; ///< Security sub-type (see indicators::ISSUE_SUB_TYPE_VALUES). + char authenticity; ///< 'P' production / 'T' test security. + char short_sale_threshold_indicator; ///< Reg SHO threshold: 'Y','N',' '. + char ipo_flag; ///< 'Y' if a new IPO, 'N' if not, ' ' if not available. + char luld_ref; ///< LULD reference price tier. + char etp_flag; ///< 'Y' if an exchange-traded product, else 'N'. + uint32_t etp_leverage_factor; ///< Leverage factor of the ETP (if applicable). + char inverse_indicator; ///< 'Y' if the ETP is an inverse product. }; /// @brief Stock Trading Action (`H`): a change in the trading status of a @@ -82,7 +82,7 @@ struct StockTradingActionMessage { char stock[8]; ///< Stock symbol, right padded with spaces. char trading_state; ///< 'H','P','Q','T' (see indicators::TRADING_STATES). char reserved; ///< Reserved. - char reason[4]; ///< Trading-action reason code (see indicators::TRADING_ACTION_REASON_CODES). + char reason[4]; ///< Trading-action reason code (see indicators::TRADING_ACTION_REASON_CODES). }; /// @brief Reg SHO Short Sale Price Test Restriction (`Y`): the Reg SHO short-sale @@ -157,14 +157,14 @@ struct MWCBStatusMessage { /// cancellation or postponement of the IPO is signalled by setting both the /// release time and the price to zero. struct IPOQuotingPeriodUpdateMessage { - char message_type = 'K'; ///< Always 'K'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - char stock[8]; ///< Stock symbol, right padded with spaces. - uint32_t ipo_quotation_release_time; ///< Seconds past midnight of the anticipated release. - char ipo_quotation_release_qualifier; ///< 'A' anticipated, 'C' cancelled/postponed. - uint32_t ipo_price; ///< IPO price (4 implied decimals). + char message_type = 'K'; ///< Always 'K'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t ipo_quotation_release_time; ///< Seconds past midnight of the anticipated release. + char ipo_quotation_release_qualifier; ///< 'A' anticipated, 'C' cancelled/postponed. + uint32_t ipo_price; ///< IPO price (4 implied decimals). }; /// @brief LULD Auction Collar (`J`): the auction collar thresholds for a security @@ -173,15 +173,15 @@ struct IPOQuotingPeriodUpdateMessage { /// Indicates the auction collar thresholds within which a paused security may /// reopen following a Limit-Up Limit-Down (LULD) trading pause. struct LULDAuctionCollarMessage { - char message_type = 'J'; ///< Always 'J'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - char stock[8]; ///< Stock symbol, right padded with spaces. - uint32_t auction_collar_reference_price; ///< Reference price for the collars (4 decimals). - uint32_t upper_auction_collar_price; ///< Upper auction collar price (4 decimals). - uint32_t lower_auction_collar_price; ///< Lower auction collar price (4 decimals). - uint32_t auction_collar_extension; ///< Number of collar extensions so far. + char message_type = 'J'; ///< Always 'J'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t auction_collar_reference_price; ///< Reference price for the collars (4 decimals). + uint32_t upper_auction_collar_price; ///< Upper auction collar price (4 decimals). + uint32_t lower_auction_collar_price; ///< Lower auction collar price (4 decimals). + uint32_t auction_collar_extension; ///< Number of collar extensions so far. }; /// @brief Operational Halt (`h`): an operational halt or resumption for a security @@ -192,13 +192,13 @@ struct LULDAuctionCollarMessage { /// Action: an operational halt is a venue-level interruption rather than a /// regulatory or volatility trading halt. struct OperationalHaltMessage { - char message_type = 'h'; ///< Always 'h'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - char stock[8]; ///< Stock symbol, right padded with spaces. - char market_code; ///< 'Q' Nasdaq, 'B' BX, 'X' PSX. - char operational_halt_action; ///< 'H' halted, 'T' resumed. + char message_type = 'h'; ///< Always 'h'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char market_code; ///< 'Q' Nasdaq, 'B' BX, 'X' PSX. + char operational_halt_action; ///< 'H' halted, 'T' resumed. }; /// @brief Add Order, No MPID Attribution (`A`): a new displayable order has been @@ -209,15 +209,15 @@ struct OperationalHaltMessage { /// number is used by every subsequent execute, cancel, delete, and replace /// message that acts on the order. struct AddOrderMessage { - char message_type = 'A'; ///< Always 'A'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - uint64_t order_reference_number; ///< Day-unique reference number for the order. - char buy_sell_indicator; ///< 'B' buy, 'S' sell. - uint32_t shares; ///< Displayed share quantity. - char stock[8]; ///< Stock symbol, right padded with spaces. - uint32_t price; ///< Display price (4 implied decimals). + char message_type = 'A'; ///< Always 'A'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Day-unique reference number for the order. + char buy_sell_indicator; ///< 'B' buy, 'S' sell. + uint32_t shares; ///< Displayed share quantity. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t price; ///< Display price (4 implied decimals). }; /// @brief Add Order, With MPID Attribution (`F`): like Add Order, but attributed @@ -227,16 +227,16 @@ struct AddOrderMessage { /// added it to the displayable book. It is identical to an Add Order message /// except that it also carries the attributing market participant identifier. struct AddOrderMPIDAttributionMessage { - char message_type = 'F'; ///< Always 'F'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - uint64_t order_reference_number; ///< Day-unique reference number for the order. - char buy_sell_indicator; ///< 'B' buy, 'S' sell. - uint32_t shares; ///< Displayed share quantity. - char stock[8]; ///< Stock symbol, right padded with spaces. - uint32_t price; ///< Display price (4 implied decimals). - char attribution[4]; ///< Market participant identifier (MPID). + char message_type = 'F'; ///< Always 'F'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Day-unique reference number for the order. + char buy_sell_indicator; ///< 'B' buy, 'S' sell. + uint32_t shares; ///< Displayed share quantity. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t price; ///< Display price (4 implied decimals). + char attribution[4]; ///< Market participant identifier (MPID). }; /// @brief Order Executed (`E`): a resting order was executed in whole or in part @@ -247,13 +247,13 @@ struct AddOrderMPIDAttributionMessage { /// effects are cumulative, and the order is removed from the book once it is fully /// executed. struct OrderExecutedMessage { - char message_type = 'E'; ///< Always 'E'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - uint64_t order_reference_number; ///< Reference number of the executed order. - uint32_t executed_shares; ///< Number of shares executed. - uint64_t match_number; ///< Day-unique match number for the execution. + char message_type = 'E'; ///< Always 'E'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the executed order. + uint32_t executed_shares; ///< Number of shares executed. + uint64_t match_number; ///< Day-unique match number for the execution. }; /// @brief Order Executed With Price (`C`): a resting order was executed at a price @@ -264,15 +264,15 @@ struct OrderExecutedMessage { /// be marked non-printable, in which case it is excluded from the time-and-sales /// tape (typically to be printed later in aggregate). struct OrderExecutedWithPriceMessage { - char message_type = 'C'; ///< Always 'C'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - uint64_t order_reference_number; ///< Reference number of the executed order. - uint32_t executed_shares; ///< Number of shares executed. - uint64_t match_number; ///< Day-unique match number for the execution. - char printable; ///< 'Y' if the trade is printable to the tape, else 'N'. - uint32_t execution_price; ///< Price at which the order executed (4 decimals). + char message_type = 'C'; ///< Always 'C'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the executed order. + uint32_t executed_shares; ///< Number of shares executed. + uint64_t match_number; ///< Day-unique match number for the execution. + char printable; ///< 'Y' if the trade is printable to the tape, else 'N'. + uint32_t execution_price; ///< Price at which the order executed (4 decimals). }; /// @brief Order Cancel (`X`): a partial cancellation reduced the shares of a @@ -283,12 +283,12 @@ struct OrderExecutedWithPriceMessage { /// order itself remains on the book. A full cancellation is conveyed by an Order /// Delete (`D`) message instead. struct OrderCancelMessage { - char message_type = 'X'; ///< Always 'X'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - uint64_t order_reference_number; ///< Reference number of the cancelled order. - uint32_t cancelled_shares; ///< Number of shares cancelled. + char message_type = 'X'; ///< Always 'X'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the cancelled order. + uint32_t cancelled_shares; ///< Number of shares cancelled. }; /// @brief Order Delete (`D`): a resting order was cancelled in its entirety and @@ -297,11 +297,11 @@ struct OrderCancelMessage { /// Sent when an order on the book is cancelled in full. All remaining shares /// become inaccessible and the order must be removed from the book. struct OrderDeleteMessage { - char message_type = 'D'; ///< Always 'D'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - uint64_t order_reference_number; ///< Reference number of the deleted order. + char message_type = 'D'; ///< Always 'D'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the deleted order. }; /// @brief Order Replace (`U`): a resting order was replaced with a new order at a @@ -312,14 +312,14 @@ struct OrderDeleteMessage { /// that is used for all subsequent updates. The side and security are unchanged; /// only the price and size may differ. struct OrderReplaceMessage { - char message_type = 'U'; ///< Always 'U'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - uint64_t original_order_reference_number; ///< Reference number being replaced. - uint64_t new_order_reference_number; ///< New reference number for the order. - uint32_t shares; ///< New displayed share quantity. - uint32_t price; ///< New display price (4 implied decimals). + char message_type = 'U'; ///< Always 'U'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t original_order_reference_number; ///< Reference number being replaced. + uint64_t new_order_reference_number; ///< New reference number for the order. + uint32_t shares; ///< New displayed share quantity. + uint32_t price; ///< New display price (4 implied decimals). }; /// @brief Trade, Non-Cross (`P`): an execution of a non-displayable order. It does @@ -331,16 +331,16 @@ struct OrderReplaceMessage { /// change book state; it exists so that consumers can include these executions in /// the trade tape and volume calculations. struct NonCrossTradeMessage { - char message_type = 'P'; ///< Always 'P'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - uint64_t order_reference_number; ///< Reference number of the non-displayed order. - char buy_sell_indicator; ///< 'B' buy, 'S' sell. - uint32_t shares; ///< Number of shares traded. - char stock[8]; ///< Stock symbol, right padded with spaces. - uint32_t price; ///< Trade price (4 implied decimals). - uint64_t match_number; ///< Day-unique match number for the trade. + char message_type = 'P'; ///< Always 'P'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the non-displayed order. + char buy_sell_indicator; ///< 'B' buy, 'S' sell. + uint32_t shares; ///< Number of shares traded. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t price; ///< Trade price (4 implied decimals). + uint64_t match_number; ///< Day-unique match number for the trade. }; /// @brief Cross Trade (`Q`): the result of a cross (opening, closing, halt/IPO @@ -383,19 +383,20 @@ struct BrokenTradeMessage { /// hypothetical clearing prices (far, near, and current reference price) the cross /// would produce, so participants can react before the cross executes. struct NOIIMessage { - char message_type = 'I'; ///< Always 'I'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - uint64_t paired_shares; ///< Shares paired at the current reference price. - uint64_t imbalance_shares; ///< Shares not paired (the imbalance). - char imbalance_direction; ///< 'B' buy, 'S' sell, 'N' none, 'O' insufficient, 'P' paused. - char stock[8]; ///< Stock symbol, right padded with spaces. - uint32_t far_price; ///< Cross price using only eligible interest (4 decimals). - uint32_t near_price; ///< Cross price using all interest (4 decimals). - uint32_t current_reference_price; ///< Price the cross would occur at now (4 decimals). - char cross_type; ///< 'O' open, 'C' close, 'H' halt/IPO cross. - char price_variation_indicator; ///< Variation band (see indicators::PRICE_VARIATION_INDICATOR). + char message_type = 'I'; ///< Always 'I'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t paired_shares; ///< Shares paired at the current reference price. + uint64_t imbalance_shares; ///< Shares not paired (the imbalance). + char imbalance_direction; ///< 'B' buy, 'S' sell, 'N' none, 'O' insufficient, 'P' paused. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t far_price; ///< Cross price using only eligible interest (4 decimals). + uint32_t near_price; ///< Cross price using all interest (4 decimals). + uint32_t current_reference_price; ///< Price the cross would occur at now (4 decimals). + char cross_type; ///< 'O' open, 'C' close, 'H' halt/IPO cross. + char + price_variation_indicator; ///< Variation band (see indicators::PRICE_VARIATION_INDICATOR). }; /// @brief Retail Price Improvement Indicator (`N`): the presence of retail price @@ -421,18 +422,18 @@ struct RetailPriceImprovementIndicatorMessage { /// thresholds, the price range collars, and the anticipated execution price and /// time used during the DLCR opening. struct DLCRMessage { - char message_type = 'O'; ///< Always 'O'. - uint16_t stock_locate; ///< Locate code identifying the security. - uint16_t tracking_number; ///< Nasdaq internal tracking number. - uint64_t timestamp; ///< Nanoseconds past midnight. - char stock[8]; ///< Stock symbol, right padded with spaces. - char open_eligibility_status; ///< Whether the security is eligible to open. - uint32_t minimum_allowable_price; ///< Lowest allowable cross price (4 decimals). - uint32_t maximum_allowable_price; ///< Highest allowable cross price (4 decimals). - uint32_t near_execution_price; ///< Anticipated cross price (4 decimals). - uint64_t near_execution_time; ///< Time of the anticipated cross (ns past midnight). - uint32_t lower_price_range_collar; ///< Lower price range collar (4 decimals). - uint32_t upper_price_range_collar; ///< Upper price range collar (4 decimals). + char message_type = 'O'; ///< Always 'O'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char open_eligibility_status; ///< Whether the security is eligible to open. + uint32_t minimum_allowable_price; ///< Lowest allowable cross price (4 decimals). + uint32_t maximum_allowable_price; ///< Highest allowable cross price (4 decimals). + uint32_t near_execution_price; ///< Anticipated cross price (4 decimals). + uint64_t near_execution_time; ///< Time of the anticipated cross (ns past midnight). + uint32_t lower_price_range_collar; ///< Lower price range collar (4 decimals). + uint32_t upper_price_range_collar; ///< Upper price range collar (4 decimals). }; // RESTORE DEFAULT PADDING From 29283a1bda3bdf32e30b59d87ca3a062b90ac345 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:33 +0100 Subject: [PATCH 116/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- include/itch/parser.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/itch/parser.hpp b/include/itch/parser.hpp index 817aac8..c8f33f0 100644 --- a/include/itch/parser.hpp +++ b/include/itch/parser.hpp @@ -22,8 +22,8 @@ #include #include #include -#include #include +#include #ifdef __cpp_lib_expected #include @@ -221,8 +221,8 @@ class Parser { /// /// @param data A view over the contiguous buffer containing ITCH data. /// @return All parsed messages on success, or a `ParseError` describing the failure. - [[nodiscard]] auto try_parse(std::span data) - -> std::expected, ParseError>; + [[nodiscard]] auto try_parse(std::span data + ) -> std::expected, ParseError>; #endif /// @brief Registers a callback invoked for each recoverable framing problem. From 96d5c92fcc3521bf7f37af21fd938bf967d4aa90 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:33 +0100 Subject: [PATCH 117/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- include/itch/price.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/itch/price.hpp b/include/itch/price.hpp index 3a93480..37f9c18 100644 --- a/include/itch/price.hpp +++ b/include/itch/price.hpp @@ -85,8 +85,8 @@ class BasicPrice { /// @param lhs The left-hand price. /// @param rhs The right-hand price. /// @return `true` if both prices have the same raw value. - [[nodiscard]] friend constexpr auto operator==(BasicPrice, BasicPrice) noexcept - -> bool = default; + [[nodiscard]] friend constexpr auto operator==(BasicPrice, BasicPrice) noexcept -> bool = + default; /// @brief Orders two prices of the same scale by raw value. /// /// @param lhs The left-hand price. From a73fe0c129f167eb2c55ba8cec55f1e254f7b29a Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:33 +0100 Subject: [PATCH 118/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- include/itch/transport/sequencing.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/include/itch/transport/sequencing.hpp b/include/itch/transport/sequencing.hpp index 7cf3011..b3001a8 100644 --- a/include/itch/transport/sequencing.hpp +++ b/include/itch/transport/sequencing.hpp @@ -30,23 +30,23 @@ namespace itch::transport { class RetransmitRequester { public: /// @brief Constructs a requester with no state. - RetransmitRequester() = default; + RetransmitRequester() = default; /// @brief Copy-constructs a requester. /// @param other The requester to copy. - RetransmitRequester(const RetransmitRequester&) = default; + RetransmitRequester(const RetransmitRequester&) = default; /// @brief Move-constructs a requester. /// @param other The requester to move from. - RetransmitRequester(RetransmitRequester&&) noexcept = default; + RetransmitRequester(RetransmitRequester&&) noexcept = default; /// @brief Copy-assigns a requester. /// @param other The requester to copy. /// @return Reference to this requester. - auto operator=(const RetransmitRequester&) -> RetransmitRequester& = default; + auto operator=(const RetransmitRequester&) -> RetransmitRequester& = default; /// @brief Move-assigns a requester. /// @param other The requester to move from. /// @return Reference to this requester. auto operator=(RetransmitRequester&&) noexcept -> RetransmitRequester& = default; /// @brief Destroys the requester. - virtual ~RetransmitRequester() = default; + virtual ~RetransmitRequester() = default; /// @brief Requests retransmission of `count` messages starting at sequence /// `start_sequence` for the given session. @@ -133,8 +133,8 @@ class SequenceTracker { /// @param session The session identifier to look up. /// @return The next expected sequence number, or `std::nullopt` if the /// session has not been observed. - [[nodiscard]] auto expected_next(std::string_view session) const - -> std::optional { + [[nodiscard]] auto expected_next(std::string_view session + ) const -> std::optional { const auto iter = m_expected_by_session.find(std::string {session}); if (iter == m_expected_by_session.end()) { return std::nullopt; From a122384a5cf4dd8f0b29f44d4f16813a414c52f6 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:33 +0100 Subject: [PATCH 119/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- src/book/l3_book.cpp | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/book/l3_book.cpp b/src/book/l3_book.cpp index e698ad7..6f7485d 100644 --- a/src/book/l3_book.cpp +++ b/src/book/l3_book.cpp @@ -37,18 +37,20 @@ auto L3Book::find_level(Side side, std::uint32_t price) const -> std::uint32_t { // Bids are sorted descending, asks ascending; binary search accordingly. if (side == Side::buy) { const auto iter = std::lower_bound( - levels.begin(), levels.end(), price, [](const Level& level, std::uint32_t value) { - return level.price > value; - } + levels.begin(), + levels.end(), + price, + [](const Level& level, std::uint32_t value) { return level.price > value; } ); if (iter != levels.end() && iter->price == price) { return static_cast(iter - levels.begin()); } } else { const auto iter = std::lower_bound( - levels.begin(), levels.end(), price, [](const Level& level, std::uint32_t value) { - return level.price < value; - } + levels.begin(), + levels.end(), + price, + [](const Level& level, std::uint32_t value) { return level.price < value; } ); if (iter != levels.end() && iter->price == price) { return static_cast(iter - levels.begin()); @@ -61,16 +63,17 @@ auto L3Book::find_or_create_level(Side side, std::uint32_t price) -> std::uint32 auto& levels = side_levels(side); auto position = side == Side::buy - ? std::lower_bound( + ? std::lower_bound( levels.begin(), levels.end(), price, [](const Level& level, std::uint32_t value) { return level.price > value; } ) - : std::lower_bound( - levels.begin(), levels.end(), price, [](const Level& level, std::uint32_t value) { - return level.price < value; - } + : std::lower_bound( + levels.begin(), + levels.end(), + price, + [](const Level& level, std::uint32_t value) { return level.price < value; } ); if (position != levels.end() && position->price == price) { return static_cast(position - levels.begin()); @@ -232,13 +235,11 @@ auto L3Book::depth(Side side, std::size_t max_levels) const -> std::vector result; result.reserve(count); for (std::size_t index = 0; index < count; ++index) { - result.push_back( - DepthLevel { - StandardPrice {levels[index].price}, - levels[index].total_shares, - levels[index].order_count - } - ); + result.push_back(DepthLevel { + StandardPrice {levels[index].price}, + levels[index].total_shares, + levels[index].order_count + }); } return result; } @@ -254,8 +255,7 @@ auto L3Book::orders_at(Side side, std::uint32_t price) const -> std::vector Date: Fri, 10 Jul 2026 18:46:33 +0100 Subject: [PATCH 120/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- src/parser.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/parser.cpp b/src/parser.cpp index 3c74df2..8aee50d 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -408,8 +408,8 @@ auto Parser::try_parse(std::span data, const MessageCallback& c return {}; } -auto Parser::try_parse(std::span data) - -> std::expected, ParseError> { +auto Parser::try_parse(std::span data +) -> std::expected, ParseError> { std::vector messages; messages.reserve(data.size() / average_message_size); if (auto error = parse_impl(as_char_ptr(data), data.size(), [&](const Message& msg) { From 0c087c822fd4a453229d274fa3df0f65b9ad9c22 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:34 +0100 Subject: [PATCH 121/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- src/transport/moldudp64.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transport/moldudp64.cpp b/src/transport/moldudp64.cpp index 78f6baa..4627fbd 100644 --- a/src/transport/moldudp64.cpp +++ b/src/transport/moldudp64.cpp @@ -20,8 +20,8 @@ auto read_big_endian(std::span data, std::size_t offset) -> Uin MoldUdp64Decoder::MoldUdp64Decoder(MessageCallback callback) : m_callback {std::move(callback)} {} -auto MoldUdp64Decoder::decode_packet(std::span packet) - -> std::optional { +auto MoldUdp64Decoder::decode_packet(std::span packet +) -> std::optional { if (packet.size() < HEADER_SIZE) { return std::nullopt; } From 28dd48f935f522be31cbfcb45b4859636c3b35b8 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:34 +0100 Subject: [PATCH 122/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- src/transport/soupbintcp.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transport/soupbintcp.cpp b/src/transport/soupbintcp.cpp index 87d262b..3b850db 100644 --- a/src/transport/soupbintcp.cpp +++ b/src/transport/soupbintcp.cpp @@ -67,9 +67,9 @@ auto SoupBinDecoder::feed(std::span bytes) -> void { break; // The rest of this packet has not arrived yet. } - const auto type = static_cast( - static_cast(m_buffer[offset + LENGTH_PREFIX_SIZE]) - ); + const auto type = + static_cast(static_cast(m_buffer[offset + LENGTH_PREFIX_SIZE]) + ); const std::span payload { m_buffer.data() + offset + LENGTH_PREFIX_SIZE + 1, static_cast(packet_length) - 1 From 26bd5885627bb3884dfae78548a77e4369899872 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:34 +0100 Subject: [PATCH 123/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- tests/book/test_book.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/book/test_book.cpp b/tests/book/test_book.cpp index 25f5b62..f0cccb8 100644 --- a/tests/book/test_book.cpp +++ b/tests/book/test_book.cpp @@ -138,8 +138,7 @@ TEST(BookManager, EmitsBboEventsOnlyWhenTopChanges) { manager.set_bbo_callback([&](const L3Book&, const itch::book::Bbo&) { ++bbo_events; }); manager.process(itch::Message {make_add(1, 10, 'B', 100, "AAPL", 1500000)}); // new top - manager.process( - itch::Message {make_add(1, 11, 'B', 100, "AAPL", 1400000)} + manager.process(itch::Message {make_add(1, 11, 'B', 100, "AAPL", 1400000)} ); // worse, no change EXPECT_EQ(bbo_events, 1); } From a01806458e07299e7d3ac936db9492314a58e5a7 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:34 +0100 Subject: [PATCH 124/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- tests/book/test_overlay.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/book/test_overlay.cpp b/tests/book/test_overlay.cpp index 4ff27c9..808f681 100644 --- a/tests/book/test_overlay.cpp +++ b/tests/book/test_overlay.cpp @@ -35,7 +35,8 @@ TEST(Overlay, ViewFieldsMatchEagerDecode) { // Lazy overlay decode of the same bytes. itch::overlay::AddOrderView view {}; std::uint64_t count = itch::overlay::for_each_message( - std::span {buffer}, [&](const itch::overlay::MessageView& generic) { + std::span {buffer}, + [&](const itch::overlay::MessageView& generic) { view = itch::overlay::AddOrderView {generic.data(), generic.size()}; } ); @@ -61,7 +62,8 @@ TEST(Overlay, FramesMultipleMessagesAndSkipsUnknownTypes) { std::uint64_t adds = 0; const auto total = itch::overlay::for_each_message( - std::span {buffer}, [&](const itch::overlay::MessageView& view) { + std::span {buffer}, + [&](const itch::overlay::MessageView& view) { if (view.type() == 'A') { ++adds; } From 9c286c3bcf7fbaa602f1dc7b8c6b34ac3c17613d Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:34 +0100 Subject: [PATCH 125/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- tests/test_parser.cpp | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_parser.cpp b/tests/test_parser.cpp index 5d74725..922834d 100644 --- a/tests/test_parser.cpp +++ b/tests/test_parser.cpp @@ -89,7 +89,7 @@ class ParserTest : public ::testing::Test { }; TEST_F(ParserTest, SingleValidSystemEventMessage) { - itch::SystemEventMessage msg_to_pack{}; + itch::SystemEventMessage msg_to_pack {}; // NOLINTBEGIN msg_to_pack.stock_locate = 1; msg_to_pack.tracking_number = 2; @@ -115,13 +115,13 @@ TEST_F(ParserTest, SingleValidSystemEventMessage) { } TEST_F(ParserTest, MultipleValidMessages) { - itch::SystemEventMessage msg1_to_pack{}; + itch::SystemEventMessage msg1_to_pack {}; // NOLINTBEGIN msg1_to_pack.stock_locate = 1; msg1_to_pack.timestamp = 3; msg1_to_pack.event_code = 'O'; - itch::AddOrderMessage msg2_to_pack{}; + itch::AddOrderMessage msg2_to_pack {}; msg2_to_pack.order_reference_number = 12345; msg2_to_pack.buy_sell_indicator = 'B'; msg2_to_pack.shares = 100; @@ -156,7 +156,7 @@ TEST_F(ParserTest, MultipleValidMessages) { TEST_F(ParserTest, ThrowsOnIncompletePayload) { // Create a valid message, then truncate it - std::string data = create_test_buffer(itch::SystemEventMessage{}); + std::string data = create_test_buffer(itch::SystemEventMessage {}); data.pop_back(); // Make the payload incomplete std::stringstream data_stream(data); @@ -180,8 +180,8 @@ TEST_F(ParserTest, HandlesEmptyStream) { } TEST_F(ParserTest, CallbackBasedParsing) { - std::string data = create_test_buffer(itch::SystemEventMessage{}) + - create_test_buffer(itch::AddOrderMessage{}); + std::string data = create_test_buffer(itch::SystemEventMessage {}) + + create_test_buffer(itch::AddOrderMessage {}); std::stringstream data_stream(data); size_t message_count = 0; @@ -199,9 +199,9 @@ TEST_F(ParserTest, CallbackBasedParsing) { } TEST_F(ParserTest, FilteredParsing) { - std::string data = create_test_buffer(itch::SystemEventMessage{}) + - create_test_buffer(itch::StockDirectoryMessage{}) + - create_test_buffer(itch::AddOrderMessage{}); + std::string data = create_test_buffer(itch::SystemEventMessage {}) + + create_test_buffer(itch::StockDirectoryMessage {}) + + create_test_buffer(itch::AddOrderMessage {}); std::stringstream data_stream(data); auto messages = parser.parse(data_stream, {'R'}); @@ -212,9 +212,9 @@ TEST_F(ParserTest, FilteredParsing) { TEST_F(ParserTest, SkipsZeroLengthMessage) { const std::array zero_len_msg = {'\x00', '\x00'}; - std::string data = create_test_buffer(itch::SystemEventMessage{}) + + std::string data = create_test_buffer(itch::SystemEventMessage {}) + std::string(zero_len_msg.begin(), zero_len_msg.end()) + - create_test_buffer(itch::SystemEventMessage{}); + create_test_buffer(itch::SystemEventMessage {}); std::stringstream data_stream(data); @@ -223,14 +223,14 @@ TEST_F(ParserTest, SkipsZeroLengthMessage) { } TEST_F(ParserTest, IgnoresTrailingGarbageData) { - std::string data = create_test_buffer(itch::SystemEventMessage{}) + "garbage"; + std::string data = create_test_buffer(itch::SystemEventMessage {}) + "garbage"; std::stringstream data_stream(data); // It throws because the "garbage" is interpreted as an incomplete header EXPECT_THROW(parser.parse(data_stream), std::runtime_error); // To test ignoring trailing data, ensure it's smaller than a header - std::string data2 = create_test_buffer(itch::SystemEventMessage{}) + "g"; + std::string data2 = create_test_buffer(itch::SystemEventMessage {}) + "g"; std::stringstream ss2(data2); EXPECT_THROW(parser.parse(ss2), std::runtime_error); @@ -239,8 +239,8 @@ TEST_F(ParserTest, IgnoresTrailingGarbageData) { } TEST_F(ParserTest, HandlesStreamEndingExactlyOnBoundary) { - std::string data = create_test_buffer(itch::SystemEventMessage{}) + - create_test_buffer(itch::AddOrderMessage{}); + std::string data = create_test_buffer(itch::SystemEventMessage {}) + + create_test_buffer(itch::AddOrderMessage {}); std::stringstream data_stream(data); // Should parse cleanly with no exceptions From f471798ca2b36d2703e4cde0b43d5a5592996594 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:46:34 +0100 Subject: [PATCH 126/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- tests/transport/frame_builders.hpp | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/transport/frame_builders.hpp b/tests/transport/frame_builders.hpp index b3bc6ae..af5d218 100644 --- a/tests/transport/frame_builders.hpp +++ b/tests/transport/frame_builders.hpp @@ -102,8 +102,12 @@ inline auto system_event_payload(std::uint64_t timestamp, char event_code) /// @param price The raw (unscaled) limit price. /// @return The encoded Add Order message payload. inline auto add_order_payload( - std::uint16_t locate, std::uint64_t ref, char side, std::uint32_t shares, - std::string_view symbol, std::uint32_t price + std::uint16_t locate, + std::uint64_t ref, + char side, + std::uint32_t shares, + std::string_view symbol, + std::uint32_t price ) -> std::vector { std::vector payload; payload.push_back(std::byte {'A'}); @@ -216,8 +220,8 @@ inline auto ethernet_ipv4_udp_frame(std::uint16_t dst_port, const std::vector>& frames) - -> std::vector { +inline auto classic_pcap(const std::vector>& frames +) -> std::vector { std::vector file; append_le32(file, 0xA1B2C3D4); // magic file.push_back(std::byte {2}); // version major (LE) From 46770ea0293e18648480495e34ee9d3b91631ab9 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 18:56:47 +0100 Subject: [PATCH 127/144] fix: restore python/arrow features in vcpkg.json Co-Authored-By: Claude Sonnet 5 --- vcpkg.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/vcpkg.json b/vcpkg.json index 165ef1b..f268b88 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -3,5 +3,24 @@ "gtest", "benchmark" ], + "features": { + "python": { + "description": "Build the pybind11 Python bindings", + "dependencies": [ + "pybind11" + ] + }, + "arrow": { + "description": "Enable Apache Arrow / Parquet columnar export", + "dependencies": [ + { + "name": "arrow", + "features": [ + "parquet" + ] + } + ] + } + }, "builtin-baseline": "5bf0c55239da398b8c6f450818c9e28d36bf9966" } From e4e4bafc3b6f121aac51386647bdcabe633384aa Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 19:41:42 +0100 Subject: [PATCH 128/144] refactor: split BookManager::process into per-type handlers Co-Authored-By: Claude Sonnet 5 --- include/itch/book/book_manager.hpp | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/include/itch/book/book_manager.hpp b/include/itch/book/book_manager.hpp index 5a531dc..54766b5 100644 --- a/include/itch/book/book_manager.hpp +++ b/include/itch/book/book_manager.hpp @@ -115,6 +115,50 @@ class BookManager { /// @return True if `symbol` should be tracked, false otherwise. [[nodiscard]] auto in_universe(std::string_view symbol) const -> bool; + /// @brief Adds a new order to the appropriate book from an Add Order (or + /// MPID-attributed Add Order) message. + /// @tparam AddMessage AddOrderMessage or AddOrderMPIDAttributionMessage. + /// @param add The parsed add-order message. + template + auto handle_add_order(const AddMessage& add) -> void; + + /// @brief Executes shares against a resting order and emits a trade using + /// the order's own price (Order Executed carries no price itself). + /// @param exec The parsed order-executed message. + auto handle_order_executed(const OrderExecutedMessage& exec) -> void; + + /// @brief Executes shares against a resting order at an explicit price + /// and emits a trade (Order Executed With Price). + /// @param exec The parsed order-executed-with-price message. + auto handle_order_executed_with_price(const OrderExecutedWithPriceMessage& exec) -> void; + + /// @brief Reduces a resting order's size (Order Cancel). + /// @param cancel The parsed order-cancel message. + auto handle_order_cancel(const OrderCancelMessage& cancel) -> void; + + /// @brief Removes a resting order from its book (Order Delete). + /// @param del The parsed order-delete message. + auto handle_order_delete(const OrderDeleteMessage& del) -> void; + + /// @brief Replaces a resting order with a new reference number, size, and + /// price (Order Replace). + /// @param replace The parsed order-replace message. + auto handle_order_replace(const OrderReplaceMessage& replace) -> void; + + /// @brief Extracts a trade-tape event from a non-displayed trade print + /// (Non-Cross Trade); does not alter the visible book. + /// @param trade The parsed non-cross-trade message. + auto handle_non_cross_trade(const NonCrossTradeMessage& trade) -> void; + + /// @brief Extracts a trade-tape event from a cross trade (Cross Trade). + /// @param cross The parsed cross-trade message. + auto handle_cross_trade(const CrossTradeMessage& cross) -> void; + + /// @brief Records the symbol associated with a locate code (Stock + /// Directory). + /// @param directory The parsed stock-directory message. + auto handle_stock_directory(const StockDirectoryMessage& directory) -> void; + std::vector> m_books_by_locate; ///< Indexed by locate. std::vector m_symbol_by_locate; ///< Locate -> symbol. std::unordered_set m_universe; ///< Empty == track all. From b2b66d8088f4bd15924b7ea71bcab65f51267410 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 19:41:42 +0100 Subject: [PATCH 129/144] refactor: split BookManager::process into per-type handlers Co-Authored-By: Claude Sonnet 5 --- src/book/book_manager.cpp | 257 ++++++++++++++++++++------------------ 1 file changed, 135 insertions(+), 122 deletions(-) diff --git a/src/book/book_manager.cpp b/src/book/book_manager.cpp index 7f0c86b..4569fc4 100644 --- a/src/book/book_manager.cpp +++ b/src/book/book_manager.cpp @@ -54,147 +54,160 @@ auto BookManager::emit_bbo_if_changed(BookEntry& target) -> void { } } -auto BookManager::process(const Message& message) -> void { - if (const auto* add = std::get_if(&message)) { - BookEntry* target = ensure_entry(add->stock_locate, to_string(add->stock, STOCK_LEN)); - if (target != nullptr) { - target->book.add_order( - add->order_reference_number, - to_side(add->buy_sell_indicator), - add->shares, - add->price - ); - emit_bbo_if_changed(*target); - } - return; - } - if (const auto* add = std::get_if(&message)) { - BookEntry* target = ensure_entry(add->stock_locate, to_string(add->stock, STOCK_LEN)); - if (target != nullptr) { - target->book.add_order( - add->order_reference_number, - to_side(add->buy_sell_indicator), - add->shares, - add->price - ); - emit_bbo_if_changed(*target); - } - return; - } - if (const auto* exec = std::get_if(&message)) { - BookEntry* target = entry(exec->stock_locate); - if (target == nullptr) { - return; - } - if (m_trade_callback) { - const auto price = target->book.order_price(exec->order_reference_number); - const auto side = target->book.order_side(exec->order_reference_number); - if (price.has_value()) { - Trade trade {}; - trade.timestamp = exec->timestamp; - trade.stock_locate = exec->stock_locate; - trade.symbol = target->book.symbol(); - trade.price = StandardPrice {*price}; - trade.shares = exec->executed_shares; - trade.match_number = exec->match_number; - trade.side = side.has_value() ? static_cast(*side) : '\0'; - trade.printable = true; - m_trade_callback(trade); - } - } - target->book.execute_order(exec->order_reference_number, exec->executed_shares); +template +auto BookManager::handle_add_order(const AddMessage& add) -> void { + BookEntry* target = ensure_entry(add.stock_locate, to_string(add.stock, STOCK_LEN)); + if (target != nullptr) { + target->book.add_order( + add.order_reference_number, to_side(add.buy_sell_indicator), add.shares, add.price + ); emit_bbo_if_changed(*target); + } +} + +auto BookManager::handle_order_executed(const OrderExecutedMessage& exec) -> void { + BookEntry* target = entry(exec.stock_locate); + if (target == nullptr) { return; } - if (const auto* exec = std::get_if(&message)) { - BookEntry* target = entry(exec->stock_locate); - if (target == nullptr) { - return; - } - if (m_trade_callback) { - const auto side = target->book.order_side(exec->order_reference_number); - Trade trade {}; - trade.timestamp = exec->timestamp; - trade.stock_locate = exec->stock_locate; + if (m_trade_callback) { + const auto price = target->book.order_price(exec.order_reference_number); + const auto side = target->book.order_side(exec.order_reference_number); + if (price.has_value()) { + Trade trade {}; + trade.timestamp = exec.timestamp; + trade.stock_locate = exec.stock_locate; trade.symbol = target->book.symbol(); - trade.price = StandardPrice {exec->execution_price}; - trade.shares = exec->executed_shares; - trade.match_number = exec->match_number; + trade.price = StandardPrice {*price}; + trade.shares = exec.executed_shares; + trade.match_number = exec.match_number; trade.side = side.has_value() ? static_cast(*side) : '\0'; - trade.printable = exec->printable == 'Y'; + trade.printable = true; m_trade_callback(trade); } - target->book.execute_order(exec->order_reference_number, exec->executed_shares); - emit_bbo_if_changed(*target); - return; } - if (const auto* cancel = std::get_if(&message)) { - if (BookEntry* target = entry(cancel->stock_locate)) { - target->book.reduce_order(cancel->order_reference_number, cancel->cancelled_shares); - emit_bbo_if_changed(*target); - } + target->book.execute_order(exec.order_reference_number, exec.executed_shares); + emit_bbo_if_changed(*target); +} + +auto BookManager::handle_order_executed_with_price(const OrderExecutedWithPriceMessage& exec) + -> void { + BookEntry* target = entry(exec.stock_locate); + if (target == nullptr) { return; } - if (const auto* del = std::get_if(&message)) { - if (BookEntry* target = entry(del->stock_locate)) { - target->book.delete_order(del->order_reference_number); - emit_bbo_if_changed(*target); - } - return; + if (m_trade_callback) { + const auto side = target->book.order_side(exec.order_reference_number); + Trade trade {}; + trade.timestamp = exec.timestamp; + trade.stock_locate = exec.stock_locate; + trade.symbol = target->book.symbol(); + trade.price = StandardPrice {exec.execution_price}; + trade.shares = exec.executed_shares; + trade.match_number = exec.match_number; + trade.side = side.has_value() ? static_cast(*side) : '\0'; + trade.printable = exec.printable == 'Y'; + m_trade_callback(trade); + } + target->book.execute_order(exec.order_reference_number, exec.executed_shares); + emit_bbo_if_changed(*target); +} + +auto BookManager::handle_order_cancel(const OrderCancelMessage& cancel) -> void { + if (BookEntry* target = entry(cancel.stock_locate)) { + target->book.reduce_order(cancel.order_reference_number, cancel.cancelled_shares); + emit_bbo_if_changed(*target); } - if (const auto* replace = std::get_if(&message)) { - if (BookEntry* target = entry(replace->stock_locate)) { - target->book.replace_order( - replace->original_order_reference_number, - replace->new_order_reference_number, - replace->shares, - replace->price - ); - emit_bbo_if_changed(*target); - } - return; +} + +auto BookManager::handle_order_delete(const OrderDeleteMessage& del) -> void { + if (BookEntry* target = entry(del.stock_locate)) { + target->book.delete_order(del.order_reference_number); + emit_bbo_if_changed(*target); } - if (const auto* trade = std::get_if(&message)) { - // A non-displayable order print does not alter the visible book. - if (m_trade_callback) { - Trade tape_entry {}; - tape_entry.timestamp = trade->timestamp; - tape_entry.stock_locate = trade->stock_locate; - tape_entry.symbol = to_string(trade->stock, STOCK_LEN); - tape_entry.price = StandardPrice {trade->price}; - tape_entry.shares = trade->shares; - tape_entry.match_number = trade->match_number; - tape_entry.side = trade->buy_sell_indicator; - tape_entry.printable = true; - m_trade_callback(tape_entry); - } - return; +} + +auto BookManager::handle_order_replace(const OrderReplaceMessage& replace) -> void { + if (BookEntry* target = entry(replace.stock_locate)) { + target->book.replace_order( + replace.original_order_reference_number, + replace.new_order_reference_number, + replace.shares, + replace.price + ); + emit_bbo_if_changed(*target); } - if (const auto* cross = std::get_if(&message)) { - if (m_trade_callback) { - Trade tape_entry {}; - tape_entry.timestamp = cross->timestamp; - tape_entry.stock_locate = cross->stock_locate; - tape_entry.symbol = to_string(cross->stock, STOCK_LEN); - tape_entry.price = StandardPrice {cross->cross_price}; - tape_entry.shares = cross->shares; - tape_entry.match_number = cross->match_number; - tape_entry.printable = true; - tape_entry.is_cross = true; - tape_entry.cross_type = cross->cross_type; - m_trade_callback(tape_entry); - } +} + +auto BookManager::handle_non_cross_trade(const NonCrossTradeMessage& trade) -> void { + // A non-displayable order print does not alter the visible book. + if (!m_trade_callback) { return; } - if (const auto* directory = std::get_if(&message)) { - if (directory->stock_locate >= m_symbol_by_locate.size()) { - m_symbol_by_locate.resize(static_cast(directory->stock_locate) + 1); - } - m_symbol_by_locate[directory->stock_locate] = to_string(directory->stock, STOCK_LEN); + Trade tape_entry {}; + tape_entry.timestamp = trade.timestamp; + tape_entry.stock_locate = trade.stock_locate; + tape_entry.symbol = to_string(trade.stock, STOCK_LEN); + tape_entry.price = StandardPrice {trade.price}; + tape_entry.shares = trade.shares; + tape_entry.match_number = trade.match_number; + tape_entry.side = trade.buy_sell_indicator; + tape_entry.printable = true; + m_trade_callback(tape_entry); +} + +auto BookManager::handle_cross_trade(const CrossTradeMessage& cross) -> void { + if (!m_trade_callback) { return; } + Trade tape_entry {}; + tape_entry.timestamp = cross.timestamp; + tape_entry.stock_locate = cross.stock_locate; + tape_entry.symbol = to_string(cross.stock, STOCK_LEN); + tape_entry.price = StandardPrice {cross.cross_price}; + tape_entry.shares = cross.shares; + tape_entry.match_number = cross.match_number; + tape_entry.printable = true; + tape_entry.is_cross = true; + tape_entry.cross_type = cross.cross_type; + m_trade_callback(tape_entry); +} + +auto BookManager::handle_stock_directory(const StockDirectoryMessage& directory) -> void { + if (directory.stock_locate >= m_symbol_by_locate.size()) { + m_symbol_by_locate.resize(static_cast(directory.stock_locate) + 1); + } + m_symbol_by_locate[directory.stock_locate] = to_string(directory.stock, STOCK_LEN); } +auto BookManager::process(const Message& message) -> void { + if (const auto* add = std::get_if(&message)) { + handle_add_order(*add); + } else if (const auto* add_mpid = std::get_if(&message)) { + handle_add_order(*add_mpid); + } else if (const auto* executed = std::get_if(&message)) { + handle_order_executed(*executed); + } else if (const auto* executed_with_price = + std::get_if(&message)) { + handle_order_executed_with_price(*executed_with_price); + } else if (const auto* cancel = std::get_if(&message)) { + handle_order_cancel(*cancel); + } else if (const auto* del = std::get_if(&message)) { + handle_order_delete(*del); + } else if (const auto* replace = std::get_if(&message)) { + handle_order_replace(*replace); + } else if (const auto* trade = std::get_if(&message)) { + handle_non_cross_trade(*trade); + } else if (const auto* cross = std::get_if(&message)) { + handle_cross_trade(*cross); + } else if (const auto* directory = std::get_if(&message)) { + handle_stock_directory(*directory); + } +} + +template auto BookManager::handle_add_order(const AddOrderMessage&) -> void; +template auto BookManager::handle_add_order(const AddOrderMPIDAttributionMessage&) -> void; + auto BookManager::book(std::uint16_t stock_locate) const -> const L3Book* { const BookEntry* found = entry(stock_locate); return found != nullptr ? &found->book : nullptr; From 82ad69f34b1dec8b42a09bc040d94404639cbe4b Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 19:41:42 +0100 Subject: [PATCH 130/144] style: use ranges::reverse_view for ask printing Co-Authored-By: Claude Sonnet 5 --- src/order_book.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/order_book.cpp b/src/order_book.cpp index 9e9d7be..bad4cea 100644 --- a/src/order_book.cpp +++ b/src/order_book.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -53,8 +54,8 @@ auto LimitOrderBook::print(std::ostream& out, unsigned int delay_ms) const -> vo // Asks are stored low-to-high; print them high-to-low so the best ask sits // just above the spread. - for (auto iter = m_asks.rbegin(); iter != m_asks.rend(); ++iter) { - print_level(iter->second.total_shares, iter->first, "Ask"); + for (const auto& [raw_price, level] : std::ranges::reverse_view(m_asks)) { + print_level(level.total_shares, raw_price, "Ask"); } out << MID_RULE; From 0b031c18092b9de1563a8cd5582676907d3b958a Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 19:41:43 +0100 Subject: [PATCH 131/144] fix: make ReplayEngine::replay const Co-Authored-By: Claude Sonnet 5 --- include/itch/replay.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/itch/replay.hpp b/include/itch/replay.hpp index 7504561..b093cef 100644 --- a/include/itch/replay.hpp +++ b/include/itch/replay.hpp @@ -53,7 +53,8 @@ class ReplayEngine { /// @param data A view over the contiguous buffer containing ITCH data. /// @param callback A function to be called for each successfully parsed message. /// @return The number of messages replayed. - auto replay(std::span data, const MessageCallback& callback) -> std::uint64_t; + auto replay(std::span data, const MessageCallback& callback) const + -> std::uint64_t; private: double m_speed_multiplier {1.0}; From d28b77b5ad4484a7331b26e546fba46c00352867 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 19:41:43 +0100 Subject: [PATCH 132/144] fix: make ReplayEngine::replay const Co-Authored-By: Claude Sonnet 5 --- src/replay.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replay.cpp b/src/replay.cpp index 67dec3f..d67f97d 100644 --- a/src/replay.cpp +++ b/src/replay.cpp @@ -15,7 +15,7 @@ auto timestamp_of(const Message& message) -> std::uint64_t { } // namespace -auto ReplayEngine::replay(std::span data, const MessageCallback& callback) +auto ReplayEngine::replay(std::span data, const MessageCallback& callback) const -> std::uint64_t { using clock = std::chrono::steady_clock; const bool paced = m_speed_multiplier > 0.0; From 93bbe5a9830eb6279052bb0baeab3ef05cad64ae Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 19:41:43 +0100 Subject: [PATCH 133/144] style: mark intentional empty catch NOLINT Co-Authored-By: Claude Sonnet 5 --- src/transport/moldudp64.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/transport/moldudp64.cpp b/src/transport/moldudp64.cpp index 4627fbd..a01e02c 100644 --- a/src/transport/moldudp64.cpp +++ b/src/transport/moldudp64.cpp @@ -59,11 +59,14 @@ auto MoldUdp64Decoder::decode_packet(std::span packet m_callback(message); } }; + // A truncated trailing block in a single datagram is non-fatal here; the gap + // will already have been surfaced by the sequence tracker. try_parse would + // avoid the exception, but it needs C++23's std::expected and this decoder + // must also build under C++20. try { m_parser.parse(blocks, counting_callback); + // NOLINTNEXTLINE(bugprone-empty-catch) } catch (const std::runtime_error&) { - // A truncated trailing block in a single datagram is non-fatal here; the - // gap will already have been surfaced by the sequence tracker. } return header; From c28f2122ae5887d47bf1b1a5752260ec2f2c0df7 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 19:41:43 +0100 Subject: [PATCH 134/144] style: mark intentional empty catch NOLINT Co-Authored-By: Claude Sonnet 5 --- src/transport/soupbintcp.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transport/soupbintcp.cpp b/src/transport/soupbintcp.cpp index 3b850db..dd059a8 100644 --- a/src/transport/soupbintcp.cpp +++ b/src/transport/soupbintcp.cpp @@ -146,11 +146,13 @@ auto SoupBinDecoder::decode_application_message(std::span paylo m_callback(message); } }; + // A malformed single-message payload is skipped rather than aborting the + // whole stream. try_parse would avoid the exception, but it needs C++23's + // std::expected and this decoder must also build under C++20. try { m_parser.parse(std::span {framed}, counting_callback); + // NOLINTNEXTLINE(bugprone-empty-catch) } catch (const std::runtime_error&) { - // A malformed single-message payload is skipped rather than aborting the - // whole stream. } } From 9f4ee236d20d081cfc0ee34aded2333fd66debcc Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 19:41:43 +0100 Subject: [PATCH 135/144] refactor: split PcapReader::handle_frame's UDP extraction Co-Authored-By: Claude Sonnet 5 --- include/itch/transport/pcap.hpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/include/itch/transport/pcap.hpp b/include/itch/transport/pcap.hpp index ceaf52c..c01b99b 100644 --- a/include/itch/transport/pcap.hpp +++ b/include/itch/transport/pcap.hpp @@ -112,6 +112,15 @@ class PcapReader { /// identifying how to interpret `frame`. auto handle_frame(std::span frame, std::uint32_t link_type) -> void; + /// @brief Locates the UDP payload within a network-layer (IPv4/IPv6) span, + /// applying the configured destination-port filter. + /// + /// @param network The network-layer bytes (IP header onward). + /// @return The UDP payload if `network` carries a matching UDP datagram, + /// `std::nullopt` otherwise. + [[nodiscard]] auto extract_udp_payload(std::span network) const + -> std::optional>; + MoldUdp64Decoder m_mold; std::optional m_port_filter {}; std::uint64_t m_udp_datagrams {0}; From 7b08b4727ede557ebc5303e4dc067b0d5ba3b740 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 19:41:44 +0100 Subject: [PATCH 136/144] refactor: split PcapReader::handle_frame's UDP extraction Co-Authored-By: Claude Sonnet 5 --- src/transport/pcap.cpp | 89 +++++++++++++++++++++++++----------------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/src/transport/pcap.cpp b/src/transport/pcap.cpp index 3f622ef..ba54191 100644 --- a/src/transport/pcap.cpp +++ b/src/transport/pcap.cpp @@ -48,6 +48,38 @@ constexpr std::uint32_t PCAP_MAGIC_NANO = 0xA1B23C4D; constexpr std::uint32_t PCAP_MAGIC_NANO_SWP = 0x4D3CB2A1; constexpr std::uint32_t PCAPNG_BLOCK_SHB = 0x0A0D0D0A; +// Strips a frame's link-layer header down to the network (IP) layer, or +// returns std::nullopt if the frame is too short or its link type isn't one +// we know how to unwrap. +auto strip_link_layer(std::span frame, std::uint32_t link_type) + -> std::optional> { + if (link_type == LINKTYPE_ETHERNET) { + if (frame.size() < ETHERNET_HEADER_SIZE) { + return std::nullopt; + } + std::size_t header = ETHERNET_HEADER_SIZE; + auto ethertype = read_net(frame, 12); + while (ethertype == ETHERTYPE_VLAN && frame.size() >= header + VLAN_TAG_SIZE) { + ethertype = read_net(frame, header + 2); + header += VLAN_TAG_SIZE; + } + if (ethertype != ETHERTYPE_IPV4 && ethertype != ETHERTYPE_IPV6) { + return std::nullopt; + } + return frame.subspan(header); + } + if (link_type == LINKTYPE_LINUX_SLL) { + if (frame.size() < LINUX_SLL_HEADER_SIZE) { + return std::nullopt; + } + return frame.subspan(LINUX_SLL_HEADER_SIZE); + } + if (link_type != LINKTYPE_RAW) { + return std::nullopt; // Unsupported link layer. + } + return frame; +} + } // namespace PcapReader::PcapReader(MessageCallback callback) : m_mold {std::move(callback)} {} @@ -182,33 +214,22 @@ auto PcapReader::read_pcapng(std::span capture) -> bool { } auto PcapReader::handle_frame(std::span frame, std::uint32_t link_type) -> void { - // Strip the link layer down to the network (IP) header. - std::span network = frame; - if (link_type == LINKTYPE_ETHERNET) { - if (frame.size() < ETHERNET_HEADER_SIZE) { - return; - } - std::size_t header = ETHERNET_HEADER_SIZE; - std::uint16_t ethertype = read_net(frame, 12); - while (ethertype == ETHERTYPE_VLAN && frame.size() >= header + VLAN_TAG_SIZE) { - ethertype = read_net(frame, header + 2); - header += VLAN_TAG_SIZE; - } - if (ethertype != ETHERTYPE_IPV4 && ethertype != ETHERTYPE_IPV6) { - return; - } - network = frame.subspan(header); - } else if (link_type == LINKTYPE_LINUX_SLL) { - if (frame.size() < LINUX_SLL_HEADER_SIZE) { - return; - } - network = frame.subspan(LINUX_SLL_HEADER_SIZE); - } else if (link_type != LINKTYPE_RAW) { - return; // Unsupported link layer. + const auto network = strip_link_layer(frame, link_type); + if (!network.has_value()) { + return; + } + const auto payload = extract_udp_payload(*network); + if (!payload.has_value()) { + return; } + ++m_udp_datagrams; + m_mold.decode_packet(*payload); +} +auto PcapReader::extract_udp_payload(std::span network) const + -> std::optional> { if (network.empty()) { - return; + return std::nullopt; } // Determine IP version and locate the UDP header. @@ -216,40 +237,38 @@ auto PcapReader::handle_frame(std::span frame, std::uint32_t li std::uint8_t protocol {0}; std::size_t ip_header_len {0}; if (version == 4) { - ip_header_len = (static_cast(network[0]) & 0x0F) * 4U; + ip_header_len = static_cast(static_cast(network[0]) & 0x0F) * 4U; if (network.size() < ip_header_len || ip_header_len < 20) { - return; + return std::nullopt; } protocol = static_cast(network[9]); } else if (version == 6) { if (network.size() < IPV6_HEADER_SIZE) { - return; + return std::nullopt; } protocol = static_cast(network[6]); ip_header_len = IPV6_HEADER_SIZE; } else { - return; + return std::nullopt; } if (protocol != IP_PROTOCOL_UDP) { - return; + return std::nullopt; } const std::span udp = network.subspan(ip_header_len); if (udp.size() < UDP_HEADER_SIZE) { - return; + return std::nullopt; } const auto dst_port = read_net(udp, 2); if (m_port_filter.has_value() && dst_port != *m_port_filter) { - return; + return std::nullopt; } - auto udp_length = read_net(udp, 4); + const auto udp_length = read_net(udp, 4); std::size_t payload_len = udp.size() - UDP_HEADER_SIZE; if (udp_length >= UDP_HEADER_SIZE) { payload_len = std::min(payload_len, udp_length - UDP_HEADER_SIZE); } - - ++m_udp_datagrams; - m_mold.decode_packet(udp.subspan(UDP_HEADER_SIZE, payload_len)); + return udp.subspan(UDP_HEADER_SIZE, payload_len); } } // namespace itch::transport From 134200276580e6c93590992071ed0854f436ad3d Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 19:41:44 +0100 Subject: [PATCH 137/144] style: designated initializers, NOLINT lower_bound Co-Authored-By: Claude Sonnet 5 --- src/book/l3_book.cpp | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/book/l3_book.cpp b/src/book/l3_book.cpp index 6f7485d..fb65d7c 100644 --- a/src/book/l3_book.cpp +++ b/src/book/l3_book.cpp @@ -35,7 +35,11 @@ auto L3Book::free_node(std::uint32_t node_index) -> void { auto L3Book::find_level(Side side, std::uint32_t price) const -> std::uint32_t { const auto& levels = side_levels(side); // Bids are sorted descending, asks ascending; binary search accordingly. + // std::ranges::lower_bound would be the modern spelling, but it trips a + // known Clang/libstdc++ interop gap on some toolchains, so this sticks + // with the plain iterator-pair form for portability. if (side == Side::buy) { + // NOLINTNEXTLINE(modernize-use-ranges) const auto iter = std::lower_bound( levels.begin(), levels.end(), @@ -46,6 +50,7 @@ auto L3Book::find_level(Side side, std::uint32_t price) const -> std::uint32_t { return static_cast(iter - levels.begin()); } } else { + // NOLINTNEXTLINE(modernize-use-ranges) const auto iter = std::lower_bound( levels.begin(), levels.end(), @@ -61,15 +66,16 @@ auto L3Book::find_level(Side side, std::uint32_t price) const -> std::uint32_t { auto L3Book::find_or_create_level(Side side, std::uint32_t price) -> std::uint32_t { auto& levels = side_levels(side); - auto position = + // See find_level's comment above on why these stay iterator-pair calls. + auto position = side == Side::buy - ? std::lower_bound( + ? std::lower_bound( // NOLINT(modernize-use-ranges) levels.begin(), levels.end(), price, [](const Level& level, std::uint32_t value) { return level.price > value; } ) - : std::lower_bound( + : std::lower_bound( // NOLINT(modernize-use-ranges) levels.begin(), levels.end(), price, @@ -236,9 +242,9 @@ auto L3Book::depth(Side side, std::size_t max_levels) const -> std::vector std::vector Date: Fri, 10 Jul 2026 19:41:44 +0100 Subject: [PATCH 138/144] style: NOLINT contains, needs C++23 Co-Authored-By: Claude Sonnet 5 --- src/io/csv_sink.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/io/csv_sink.cpp b/src/io/csv_sink.cpp index c3c483a..69e00ca 100644 --- a/src/io/csv_sink.cpp +++ b/src/io/csv_sink.cpp @@ -37,6 +37,9 @@ auto fill_common(Row& row, const MsgType& msg) -> void { // Quotes a value only if it contains a comma; symbols never do, but be safe. auto field(const std::string& value) -> std::string { + // std::string::contains needs C++23, but this file must also build under + // this project's C++20 floor. + // NOLINTNEXTLINE(readability-container-contains) if (value.find(',') != std::string::npos) { return std::format("\"{}\"", value); } From 36c8a931c498cbed6e2fd6a5753b49e116c4eebf Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 22:27:18 +0100 Subject: [PATCH 139/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- src/book/book_manager.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/book/book_manager.cpp b/src/book/book_manager.cpp index 4569fc4..f67335c 100644 --- a/src/book/book_manager.cpp +++ b/src/book/book_manager.cpp @@ -90,8 +90,8 @@ auto BookManager::handle_order_executed(const OrderExecutedMessage& exec) -> voi emit_bbo_if_changed(*target); } -auto BookManager::handle_order_executed_with_price(const OrderExecutedWithPriceMessage& exec) - -> void { +auto BookManager::handle_order_executed_with_price(const OrderExecutedWithPriceMessage& exec +) -> void { BookEntry* target = entry(exec.stock_locate); if (target == nullptr) { return; From 3b30558ac29c345eb84533ec2792969364adc48d Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 22:27:18 +0100 Subject: [PATCH 140/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- src/book/l3_book.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/book/l3_book.cpp b/src/book/l3_book.cpp index fb65d7c..a044028 100644 --- a/src/book/l3_book.cpp +++ b/src/book/l3_book.cpp @@ -69,13 +69,13 @@ auto L3Book::find_or_create_level(Side side, std::uint32_t price) -> std::uint32 // See find_level's comment above on why these stay iterator-pair calls. auto position = side == Side::buy - ? std::lower_bound( // NOLINT(modernize-use-ranges) + ? std::lower_bound( // NOLINT(modernize-use-ranges) levels.begin(), levels.end(), price, [](const Level& level, std::uint32_t value) { return level.price > value; } ) - : std::lower_bound( // NOLINT(modernize-use-ranges) + : std::lower_bound( // NOLINT(modernize-use-ranges) levels.begin(), levels.end(), price, From b85b20cb8b44419efa4b1bfeb220a3d094961e74 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 22:27:18 +0100 Subject: [PATCH 141/144] style: reformat to clang-format 18 Co-Authored-By: Claude Sonnet 5 --- include/itch/transport/pcap.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/itch/transport/pcap.hpp b/include/itch/transport/pcap.hpp index c01b99b..b136b7d 100644 --- a/include/itch/transport/pcap.hpp +++ b/include/itch/transport/pcap.hpp @@ -118,8 +118,8 @@ class PcapReader { /// @param network The network-layer bytes (IP header onward). /// @return The UDP payload if `network` carries a matching UDP datagram, /// `std::nullopt` otherwise. - [[nodiscard]] auto extract_udp_payload(std::span network) const - -> std::optional>; + [[nodiscard]] auto extract_udp_payload(std::span network + ) const -> std::optional>; MoldUdp64Decoder m_mold; std::optional m_port_filter {}; From d23142f58a5f545a1f1f8dad18a3ceb0c5d1e66d Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 22:27:18 +0100 Subject: [PATCH 142/144] fix: heap-buffer-overflow reading truncated pcapng EPB blocks Co-Authored-By: Claude Sonnet 5 --- src/transport/pcap.cpp | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/transport/pcap.cpp b/src/transport/pcap.cpp index ba54191..ac3cff1 100644 --- a/src/transport/pcap.cpp +++ b/src/transport/pcap.cpp @@ -185,15 +185,23 @@ auto PcapReader::read_pcapng(std::span capture) -> bool { constexpr std::size_t EPB_IFACE_OFFSET = BLOCK_HEADER_SIZE; constexpr std::size_t EPB_CAPLEN_OFFSET = BLOCK_HEADER_SIZE + 12; constexpr std::size_t EPB_DATA_OFFSET = BLOCK_HEADER_SIZE + 20; - const auto iface = - read_le_or_be(capture, offset + EPB_IFACE_OFFSET, swapped); - const auto cap_len = - read_le_or_be(capture, offset + EPB_CAPLEN_OFFSET, swapped); - if (offset + EPB_DATA_OFFSET + cap_len <= capture.size() && - iface < interface_link_types.size()) { - handle_frame( - capture.subspan(offset + EPB_DATA_OFFSET, cap_len), interface_link_types[iface] - ); + // The block must be large enough to hold its fixed header fields + // (interface id, two timestamp words, captured/packet length) plus + // the trailing repeated total-length word before those fields can + // be read; total_length alone (checked above) only guarantees the + // block fits in the buffer, not that it's this shape. + if (total_length >= EPB_DATA_OFFSET + sizeof(std::uint32_t)) { + const auto iface = + read_le_or_be(capture, offset + EPB_IFACE_OFFSET, swapped); + const auto cap_len = + read_le_or_be(capture, offset + EPB_CAPLEN_OFFSET, swapped); + if (offset + EPB_DATA_OFFSET + cap_len <= capture.size() && + iface < interface_link_types.size()) { + handle_frame( + capture.subspan(offset + EPB_DATA_OFFSET, cap_len), + interface_link_types[iface] + ); + } } } else if (block_type == BLOCK_SPB) { constexpr std::size_t SPB_DATA_OFFSET = BLOCK_HEADER_SIZE + 4; @@ -226,8 +234,8 @@ auto PcapReader::handle_frame(std::span frame, std::uint32_t li m_mold.decode_packet(*payload); } -auto PcapReader::extract_udp_payload(std::span network) const - -> std::optional> { +auto PcapReader::extract_udp_payload(std::span network +) const -> std::optional> { if (network.empty()) { return std::nullopt; } From 40e319e4f26ab53a37a067a9645744af3e765ee2 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 22:34:25 +0100 Subject: [PATCH 143/144] ci: install autotools deps for vcpkg's python3 port build Co-Authored-By: Claude Sonnet 5 --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03a2aec..de88302 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -275,6 +275,9 @@ jobs: with: python-version: "3.12" + - name: Install vcpkg python3 port build dependencies + run: sudo apt-get update && sudo apt-get install -y autoconf autoconf-archive automake libtool + - name: Cache vcpkg uses: actions/cache@v4 with: From 402c038a5adc5d4176d947403dd510bb2525e644 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Fri, 10 Jul 2026 23:20:22 +0100 Subject: [PATCH 144/144] fix: build itch with -fPIC so it links into the Python .so Co-Authored-By: Claude Sonnet 5 --- src/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3312848..b74fcff 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -39,6 +39,11 @@ endif() configure_visibility(itch) +# itch is a static library by default, but the pybind11 module (and any other +# shared-library consumer) links its object code into a .so, which requires +# position-independent code regardless of how itch itself is built. +set_target_properties(itch PROPERTIES POSITION_INDEPENDENT_CODE ON) + # Step 2: Update Include Directories target_include_directories( itch PUBLIC