diff --git a/.clang-tidy b/.clang-tidy index 92c0be5..444c6a8 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -22,12 +22,16 @@ Checks: > google-*, -cppcoreguidelines-pro-bounds-pointer-arithmetic, -cppcoreguidelines-pro-bounds-array-to-pointer-decay, + -cppcoreguidelines-pro-bounds-constant-array-index, + -cppcoreguidelines-pro-bounds-avoid-unchecked-container-access, -cppcoreguidelines-owning-memory, -hicpp-signed-bitwise, -modernize-use-trailing-return-type, -modernize-use-nodiscard, -readability-identifier-naming, -readability-magic-numbers, + -cppcoreguidelines-avoid-magic-numbers, + -bugprone-easily-swappable-parameters, -google-build-using-namespace, -google-runtime-references, -cert-err58-cpp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c098c5c..de88302 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,8 @@ name: build on: push: branches: [ "main" ] + pull_request: + branches: [ "main" ] jobs: build-and-test: @@ -42,7 +44,7 @@ jobs: key: ${{ runner.os }}-${{ matrix.build_type }}-cmake-${{ hashFiles('CMakeLists.txt', '**/CMakeLists.txt') }} restore-keys: | ${{ runner.os }}-${{ matrix.build_type }}-cmake- - + - name: Set VCPKG Triplet shell: bash run: | @@ -64,6 +66,8 @@ jobs: cmake -S . -B $BUILD_DIR \ -DITCH_PROJECT_ENV=PROD \ -DITCH_BUILD_TESTS=ON \ + -DITCH_BUILD_TOOLS=ON \ + -DITCH_BUILD_EXAMPLES=ON \ -DITCH_CXX_STANDARD=${{ matrix.cpp_standard }} \ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ @@ -72,7 +76,289 @@ jobs: - name: Build shell: bash run: cmake --build $BUILD_DIR --config ${{ matrix.build_type }} - + - name: Run tests shell: bash run: ctest --test-dir $BUILD_DIR --output-on-failure --build-config ${{ matrix.build_type }} + + format-check: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Install clang-format + run: pip install clang-format==18.1.8 + + - name: Check formatting + run: | + find src include tests benchmarks examples -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.h' \) \ + -print0 | xargs -0 clang-format --dry-run --Werror + + clang-tidy: + runs-on: ubuntu-latest + env: + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Install clang-tidy + run: sudo apt-get update && sudo apt-get install -y clang-tidy + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + ${{ env.VCPKG_ROOT }} + ~/.cache/vcpkg + key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg- + + - name: Set up vcpkg + uses: lukka/run-vcpkg@v11 + + - name: Configure CMake (export compile commands) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_TESTS=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-linux + + - name: Run clang-tidy + run: | + find src -name '*.cpp' -print0 \ + | xargs -0 clang-tidy -p "$BUILD_DIR" + + sanitizers: + runs-on: ubuntu-latest + env: + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + ${{ env.VCPKG_ROOT }} + ~/.cache/vcpkg + key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg- + + - name: Set up vcpkg + uses: lukka/run-vcpkg@v11 + + - name: Configure CMake (ASAN + UBSAN) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_TESTS=ON \ + -DITCH_ENABLE_ADDRESS_SANITIZER=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_FLAGS="-fsanitize=undefined -fno-sanitize=alignment -fno-sanitize-recover=undefined" \ + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=undefined" \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-linux + + - name: Build + run: cmake --build $BUILD_DIR + + - name: Run tests under sanitizers + run: ctest --test-dir $BUILD_DIR --output-on-failure + + coverage: + runs-on: ubuntu-latest + env: + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Install gcovr + run: sudo apt-get update && sudo apt-get install -y gcovr + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + ${{ env.VCPKG_ROOT }} + ~/.cache/vcpkg + key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg- + + - name: Set up vcpkg + uses: lukka/run-vcpkg@v11 + + - name: Configure CMake (coverage) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_TESTS=ON \ + -DITCH_ADD_COVERAGE_ANALYSIS=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-linux + + - name: Build + run: cmake --build $BUILD_DIR + + - name: Run tests + run: ctest --test-dir $BUILD_DIR --output-on-failure + + - name: Generate coverage report + run: cmake --build $BUILD_DIR --target coverage + + - name: Upload coverage report + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: ${{ env.BUILD_DIR }}/coverage.html + if-no-files-found: ignore + + fuzz: + runs-on: ubuntu-latest + env: + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Install clang + run: sudo apt-get update && sudo apt-get install -y clang + + - name: Configure CMake (fuzzers) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_FUZZERS=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ + + - name: Build fuzzers + run: cmake --build $BUILD_DIR --target parser_fuzzer moldudp64_fuzzer + + - name: Run parser fuzzer (time-budgeted) + run: $BUILD_DIR/fuzz/parser_fuzzer -max_total_time=60 -max_len=512 -print_final_stats=1 + + - name: Run transport fuzzer (time-budgeted) + run: $BUILD_DIR/fuzz/moldudp64_fuzzer -max_total_time=60 -max_len=2048 -print_final_stats=1 + + python-bindings: + runs-on: ubuntu-latest + env: + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install vcpkg python3 port build dependencies + run: sudo apt-get update && sudo apt-get install -y autoconf autoconf-archive automake libtool + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + ${{ env.VCPKG_ROOT }} + ~/.cache/vcpkg + key: ${{ runner.os }}-vcpkg-python-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg- + + - name: Set up vcpkg + uses: lukka/run-vcpkg@v11 + + - name: Configure CMake (Python bindings) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_PYTHON=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Release \ + -DPython_EXECUTABLE=$(which python) \ + -DVCPKG_MANIFEST_FEATURES=python \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-linux + + - name: Build extension + run: cmake --build $BUILD_DIR --target itchcpp_python + + - name: Run binding tests + run: | + python -m pip install --upgrade pip pytest + # Drop the freshly built extension into the package so `import itchcpp` + # resolves to python/itchcpp with its compiled _itchcpp module. + cp $BUILD_DIR/python/_itchcpp*.so python/itchcpp/ + cd python && python -m pytest tests -q + + benchmark-smoke: + runs-on: ubuntu-latest + env: + VCPKG_ROOT: ${{ github.workspace }}/vcpkg + BUILD_DIR: ${{ github.workspace }}/build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Cache vcpkg + uses: actions/cache@v4 + with: + path: | + ${{ env.VCPKG_ROOT }} + ~/.cache/vcpkg + key: ${{ runner.os }}-vcpkg-${{ hashFiles('vcpkg.json') }} + restore-keys: | + ${{ runner.os }}-vcpkg- + + - name: Set up vcpkg + uses: lukka/run-vcpkg@v11 + + - name: Configure CMake (benchmarks) + run: | + cmake -S . -B $BUILD_DIR \ + -DITCH_BUILD_BENCHMARKS=ON \ + -DITCH_CXX_STANDARD=23 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=x64-linux + + - name: Build benchmarks + run: cmake --build $BUILD_DIR --target parser_bench book_bench + + - name: Generate a small synthetic ITCH sample + run: | + python3 - <<'PY' + frame = (bytes([0x00, 0x0C]) + b'S' + + (1).to_bytes(2, 'big') + (2).to_bytes(2, 'big') + + (3).to_bytes(6, 'big') + b'O') + with open('sample.itch', 'wb') as out: + out.write(frame * 1_000_000) + PY + + - name: Run benchmarks (smoke) + run: | + $BUILD_DIR/benchmarks/parser_bench sample.itch --benchmark_min_time=1x + $BUILD_DIR/benchmarks/book_bench sample.itch --benchmark_min_time=1x diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..e676fb8 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,47 @@ +name: docs + +on: + push: + branches: ["main"] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v4 + + - name: Install Doxygen and Graphviz + run: sudo apt-get update && sudo apt-get install -y doxygen graphviz + + - name: Build documentation + run: | + cmake -S . -B build -DITCH_BUILD_DOCUMENTATION=ON + cmake --build build --target docs + touch docs/html/.nojekyll + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/html + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..c0c190a --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,73 @@ +name: Publish + +on: + push: + tags: + - "v*" + workflow_dispatch: + +jobs: + build_wheels: + name: Wheels (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-13, macos-14] + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.3 + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: ./wheelhouse/*.whl + + build_sdist: + name: Source distribution + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Build sdist + run: pipx run build --sdist + + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + publish: + name: Publish to PyPI and create the release + needs: [build_wheels, build_sdist] + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + environment: + name: pypi + url: https://pypi.org/project/itchcpp/ + permissions: + contents: write + id-token: write + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + files: dist/* + generate_release_notes: true diff --git a/.gitignore b/.gitignore index 5abe957..1c7eee4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,36 @@ *.gz *.pdf build/ +build-*/ install/ +scratch_* +*.itch *-workspace *_ITCH50 scripts.sh *Presets.json .cache/ -/packaging +prompts.txt + +# Generated coverage reports must not be committed; they belong in the build tree. +coverage.html +docs/coverage.html + +# Python build artifacts. The compiled extension (itchcpp/_itchcpp*.pyd|.so) is a +# build output; the hand-written .pyi stub beside it is source and is kept. +__pycache__/ +*.pyd +*.so +*.egg-info/ +.pytest_cache/ +.mypy_cache/ +dist/ +wheelhouse/ + +# Generated documentation. The doxygen-awesome-css theme and the HTML header are +# fetched/generated into the CMake build tree by build_documentation(), so only the +# rendered site under docs/html is produced inside docs/. +docs/html/ +CLAUDE.md +PROMPTS.md +DEVELOPER_GUIDE.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 0981ef1..c37a90a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,9 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) file(READ "VERSION.txt" RAW_VERSION) string(STRIP "${RAW_VERSION}" APP_VERSION) -project(ITCH VERSION ${APP_VERSION}) +project(ITCH LANGUAGES CXX VERSION ${APP_VERSION}) include(${CMAKE_SOURCE_DIR}/cmake/Helpers.cmake) include(${CMAKE_SOURCE_DIR}/cmake/Versions.cmake) @@ -12,16 +12,16 @@ if(${PROJECT_NAME}_ADD_COVERAGE_ANALYSIS) add_coverage_analysis() endif() if(${PROJECT_NAME}_APPLY_FORMATING) - apply_formating(${PROJECT_NAME}_USE_PER_FILE_FORMATTING) + apply_formatting(${PROJECT_NAME}_USE_PER_FILE_FORMATTING) endif() if(${PROJECT_NAME}_APPLY_CLANG_TIDY_GLOBALY) - apply_clang_tidy_globaly() + apply_clang_tidy_globally() endif() if(${PROJECT_NAME}_BUILD_DOCUMENTATION) - build_documenation() + build_documentation() endif() if(${PROJECT_NAME}_ENABLE_ADDRESS_SANITIZER) - anable_address_sanitizer() + enable_address_sanitizer() endif() add_subdirectory(src) @@ -36,5 +36,14 @@ endif() if (${PROJECT_NAME}_BUILD_EXAMPLES) add_subdirectory(examples) endif() +if (${PROJECT_NAME}_BUILD_FUZZERS) + add_subdirectory(fuzz) +endif() +if (${PROJECT_NAME}_BUILD_TOOLS) + add_subdirectory(tools/itch_tool) +endif() +if (${PROJECT_NAME}_BUILD_PYTHON) + add_subdirectory(python) +endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..63a2677 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,78 @@ +# Contributing to ITCHCPP + +Thanks for your interest in improving ITCHCPP. This guide covers how to build, +test, and submit changes, and the compatibility policy the project follows. + +## Getting started + +ITCHCPP targets **C++20** (and builds under C++23). You need a recent compiler +(GCC 12+, Clang 15+, or MSVC 19.3x) and CMake 3.25+. Developer dependencies +(GoogleTest, Google Benchmark, and the optional pybind11/Arrow features) are +resolved through vcpkg. + +```bash +cmake -S . -B build \ + -DITCH_BUILD_TESTS=ON -DITCH_BUILD_EXAMPLES=ON -DITCH_BUILD_TOOLS=ON \ + -DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake +cmake --build build +ctest --test-dir build --output-on-failure +``` + +Optional features are off by default and enabled with: + +- `-DITCH_BUILD_BENCHMARKS=ON` — Google Benchmark suites. +- `-DITCH_BUILD_FUZZERS=ON` — libFuzzer targets (Clang only). +- `-DITCH_BUILD_TOOLS=ON` — the `itch-tool` CLI. +- `-DITCH_BUILD_PYTHON=ON` — the pybind11 Python bindings (needs the `python` + vcpkg feature: `-DVCPKG_MANIFEST_FEATURES=python`). +- `-DITCH_WITH_ARROW=ON` — Apache Arrow / Parquet export (needs the `arrow` + vcpkg feature: `-DVCPKG_MANIFEST_FEATURES=arrow`). +- `-DITCH_BUILD_DOCUMENTATION=ON` — the Doxygen `docs` target. Build it with + `cmake --build build --target docs` (needs Doxygen; the modern theme is fetched + automatically). Output is written to `docs/html`. The same documentation is + published to GitHub Pages from `main` by `.github/workflows/docs.yml`. + +## Code style + +- Follow the rules in [CLAUDE](CLAUDE.md): `snake_case` functions/variables, + `PascalCase` types, `m_` private members, `SCREAMING_SNAKE_CASE` constants, + symbols at least three characters, trailing return types, brace initialization, + and `std::print`/`std::format` for output. +- Run `clang-format` (the repo ships a `.clang-format`) and `clang-tidy` before + submitting; CI enforces both. +- Document the _why_ in comments; use Doxygen `///` for public API. + +## Tests + +- Add tests under `tests/` mirroring the source layout, using GoogleTest. +- Cover edge and error cases, not just the happy path. Parsers and decoders that + consume untrusted input should also be exercised by a fuzz target. +- Keep tests deterministic and fast. + +## Pull requests + +- CI must pass: the build/test matrix (GCC, Clang, MSVC; C++20 and C++23), + clang-format, clang-tidy, sanitizers (ASAN/UBSAN), coverage, fuzzing, and the + Python-binding job. +- Keep the public API surface minimal; mark deprecations with + `[[deprecated("reason")]]` and provide a migration path. +- Update [CHANGELOG](CHANGELOG.md) under `[Unreleased]` and follow the Boy + Scout Rule. + +## Versioning and compatibility policy + +ITCHCPP follows [Semantic Versioning 2.0.0](https://semver.org). A breaking change +is any backward-incompatible modification to the public API, which for this project +includes: + +- the C++ headers under `include/itch/` (types, function signatures, struct + layouts of the wire message structs), +- the message wire formats and the encoder/parser round-trip contract, +- the `itch-tool` command-line interface, +- the Python module's public API. + +The build options and CMake target names (`itch::itch`) are also part of the +public surface. Within a major version, source compatibility is preserved; the +binary (ABI) is **not** guaranteed stable across minor versions, so link against a +single version. New venues/protocols are added behind the `itch::venue::VenuePolicy` +seam without breaking existing policies. diff --git a/README.md b/README.md index 7ccad7c..f64d15c 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,13 @@ - # High-Performance NASDAQ ITCH 5.0 Parser [![Build](https://github.com/bbalouki/itchcpp/actions/workflows/build.yml/badge.svg)](https://github.com/bbalouki/itchcpp/actions/workflows/build.yml) +[![Docs](https://github.com/bbalouki/itchcpp/actions/workflows/docs.yml/badge.svg)](https://bbalouki.github.io/itchcpp/) ![Linux](https://img.shields.io/badge/Linux-FCC624?logo=linux&logoColor=black) ![macOS](https://img.shields.io/badge/macOS-000000?logo=apple&logoColor=white) ![Windows](https://img.shields.io/badge/Windows-0078D6?logo=windows&logoColor=white) +![Vcpkg Version](https://img.shields.io/vcpkg/v/bbalouki-itch) [![C++20](https://img.shields.io/badge/C++-20-blue.svg)](https://isocpp.org/std/the-standard) -[![CMake](https://img.shields.io/badge/CMake-3.20+-blue.svg)](https://cmake.org/) +[![CMake](https://img.shields.io/badge/CMake-3.25+-blue.svg)](https://cmake.org/) [![CodeFactor](https://www.codefactor.io/repository/github/bbalouki/itchcpp/badge)](https://www.codefactor.io/repository/github/bbalouki/itchcpp) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![LinkedIn](https://img.shields.io/badge/LinkedIn-grey?logo=Linkedin&logoColor=white)](https://www.linkedin.com/in/bertin-balouki-simyeli-15b17a1a6/) @@ -14,6 +15,8 @@ A modern, high-performance C++20 library for parsing NASDAQ TotalView-ITCH 5.0 protocol data feeds. This parser is designed for maximum speed, minimal memory overhead, and type safety, making it ideal for latency-sensitive financial applications, market data analysis, and quantitative research. +📖 [**API documentation:**](https://bbalouki.github.io/itchcpp/). + --- ## Table of Contents @@ -21,25 +24,25 @@ A modern, high-performance C++20 library for parsing NASDAQ TotalView-ITCH 5.0 p 1. [Project Philosophy](#project-philosophy) 2. [Key Features](#key-features) 3. [Architectural Overview](#architectural-overview) - * [Zero-Overhead Deserialization](#zero-overhead-deserialization) - * [Type-Safe Message Handling](#type-safe-message-handling) - * [Endianness and Byte Order](#endianness-and-byte-order) + - [Fast, Allocation-Free Single-Pass Decoding](#fast-allocation-free-single-pass-decoding) + - [Type-Safe Message Handling](#type-safe-message-handling) + - [Endianness and Byte Order](#endianness-and-byte-order) 4. [Getting Started](#getting-started) - * [Prerequisites](#prerequisites) - * [Building the Project](#building-the-project) - * [Running Tests and Benchmarks](#running-tests-and-benchmarks) + - [Prerequisites](#prerequisites) + - [Building the Project](#building-the-project) + - [Running Tests and Benchmarks](#running-tests-and-benchmarks) 5. [Dependency Management](#dependency-management) 6. [Comprehensive Usage Guide](#comprehensive-usage-guide) - * [Example 1: Parsing a File into a Vector](#example-1-parsing-a-file-into-a-vector) - * [Example 2: High-Performance Callback-Based Parsing](#example-2-high-performance-callback-based-parsing) - * [Example 3: Filtering Messages by Type](#example-3-filtering-messages-by-type) - * [Example 4: A Complete, Compilable Example](#example-4-a-complete-compilable-example) + - [Example 1: Parsing a File into a Vector](#example-1-parsing-a-file-into-a-vector) + - [Example 2: High-Performance Callback-Based Parsing](#example-2-high-performance-callback-based-parsing) + - [Example 3: Filtering Messages by Type](#example-3-filtering-messages-by-type) + - [Example 4: A Complete, Compilable Example](#example-4-a-complete-compilable-example) 7. [API Reference: ITCH 5.0 Message Types](#api-reference-itch-50-message-types) - * [System Event Messages](#system-event-messages) - * [Stock and Administrative Messages](#stock-and-administrative-messages) - * [Order Messages](#order-messages) - * [Trade Messages](#trade-messages) - * [Imbalance and Price Discovery Messages](#imbalance-and-price-discovery-messages) + - [System Event Messages](#system-event-messages) + - [Stock and Administrative Messages](#stock-and-administrative-messages) + - [Order Messages](#order-messages) + - [Trade Messages](#trade-messages) + - [Imbalance and Price Discovery Messages](#imbalance-and-price-discovery-messages) 8. [Performance](#performance) 9. [Coding Standards](#coding-standards) 10. [Frequently Asked Questions (FAQ)](#frequently-asked-questions-faq) @@ -53,7 +56,7 @@ A modern, high-performance C++20 library for parsing NASDAQ TotalView-ITCH 5.0 p The design of this ITCH parser is guided by three principles: -1. **Performance Above All**: In market data processing, performance is not just a feature; it's a requirement. This library is architected to eliminate unnecessary overhead. It avoids dynamic memory allocations, data copies, and stream I/O in its critical path. The goal is to convert raw binary data into a usable C++ struct as efficiently as possible. +1. **Performance Above All**: In market data processing, performance is not just a feature; it's a requirement. This library is architected to eliminate unnecessary overhead. It avoids dynamic memory allocations and stream I/O in its critical path, decoding each frame in a single forward pass. The goal is to convert raw binary data into a usable C++ struct as efficiently as possible. 2. **Correctness and Safety**: Financial data is unforgiving. A single misinterpreted byte can corrupt analysis. We prioritize correctness by adhering strictly to the ITCH 5.0 specification and ensuring data is represented in a type-safe manner. @@ -63,16 +66,36 @@ The design of this ITCH parser is guided by three principles: ## Key Features -* **High Throughput**: Capable of multi-gigabyte-per-second parsing speeds on modern hardware. -* **Zero-Allocation Core**: The primary parsing loop performs zero dynamic memory allocations, minimizing latency and jitter. -* **Type-Safe API**: All ITCH messages are deserialized into a `std::variant` of dedicated, packed `struct`s, ensuring compile-time safety. -* **Specification Compliant**: Faithfully implements the message formats outlined in the Nasdaq TotalView-ITCH 5.0 specification. -* **Flexible Parsing Strategies**: - * Eagerly parse a buffer into a `std::vector` for convenience. - * Use a callback for efficient, low-memory stream processing of large files. - * Filter messages by type at the parsing stage to reduce processing load. -* **Cross-Platform**: Compatible with any platform supporting a C++20 compiler and CMake. -* **Minimal Dependencies**: Zero runtime dependencies. Test and benchmark libraries are managed by `vcpkg`. +- **Real Feed Ingestion**: Consume ITCH as it actually arrives, MoldUDP64 UDP + multicast framing, SoupBinTCP (Glimpse/recovery) framing, and `.pcap`/`.pcapng` + captures, with per-session sequence tracking and gap detection. No libpcap + dependency. See [Feed Ingestion](#feed-ingestion-transport). +- **Full-Market Book Engine**: Reconstruct every symbol on the feed in one pass + with an allocation-light L3 order book (object pool, intrusive FIFO levels, flat + ladders, open-addressed O(1) order lookup), with BBO change events, L2/L3 depth + snapshots, and trade-tape extraction. See [Book Engine](#full-market-book-engine). +- **Zero-Copy Overlay API**: Inspect raw frames through lazy typed views that + convert only the fields you read, for hot paths that touch a few fields per + message. +- **Built-in Analytics**: Header-only microstructure layer, OHLCV bar builders + (time/tick/volume clocks), VWAP/TWAP, spread, queue imbalance, order-flow + imbalance, NOII surfacing, and auction reconstruction. See [Analytics](#analytics). +- **Interoperability**: CSV and Arrow/Parquet export, a batteries-included + `itch-tool` CLI (inspect/filter/stats/convert), and native Python bindings + (pybind11). See [Interoperability](#interoperability). +- **Simulation & Ecosystem**: Timestamp-paced replay engine, a full ITCH + encoder/writer (`parse(encode(msg)) == msg`), multi-venue extension seams, and + Conan/vcpkg packaging. See [Simulation](#simulation--ecosystem). +- **High Throughput**: Multi-gigabyte-per-second parsing on modern hardware (see [Benchmarks](#benchmarks) for measured numbers). +- **Allocation-Free Core**: The callback-based parsing loop performs zero dynamic memory allocations, minimizing latency and jitter. +- **Type-Safe API**: All ITCH messages are deserialized into a `std::variant` of dedicated, packed `struct`s, ensuring compile-time safety. +- **Specification Compliant**: Faithfully implements the message formats outlined in the Nasdaq TotalView-ITCH 5.0 specification. +- **Flexible Parsing Strategies**: + - Eagerly parse a buffer into a `std::vector` for convenience. + - Use a callback for efficient, low-memory stream processing of large files. + - Filter messages by type at the parsing stage to reduce processing load. +- **Cross-Platform**: Compatible with any platform supporting a C++20 compiler and CMake. +- **Minimal Dependencies**: Zero runtime dependencies. Test and benchmark libraries are managed by `vcpkg`. --- @@ -80,17 +103,21 @@ The design of this ITCH parser is guided by three principles: The library is built on some key principles to deliver maximum performance and safety. -### Zero-Overhead Deserialization +### Fast, Allocation-Free Single-Pass Decoding + +The parser is a fast, allocation-free, single-pass field decoder. Each message type is defined as a packed `struct` whose layout mirrors the ITCH wire format, and each frame is decoded in one forward pass: fields are copied out and converted from network byte order directly into the typed `struct`, with no intermediate buffers and no per-message heap allocation on the callback path. -To achieve its performance, the parser uses a zero-overhead deserialization strategy. The parser achieves its speed by reading messages directly from the raw data stream without performing heavy data conversions. Each message type is defined in a compact format that matches exactly how the data appears on the network. This means the parser can quickly copy incoming bytes into a message structure in memory, skipping the usual per-field decoding steps. The result is near-zero overhead when reading and processing messages. +Dispatch is driven by a compile-time table keyed on the one-byte message type, so selecting the right decoder is a single flat-array lookup followed by a direct, inlinable call. There is no red-black-tree lookup and no type-erased `std::function` indirection. The frame length declared on the wire is validated against the size the message type requires before any field is read, so a truncated or malformed frame can never cause the decoder to read into an adjacent message. + +This is genuine single-pass decoding rather than a zero-copy overlay: the bytes are converted eagerly into host-order fields. ### Type-Safe Message Handling All possible ITCH messages are handled using a single, modern C++ object: `itch::Message`, which is a `std::variant`. This approach provides significant advantages over traditional designs: -* **Compile-Time Safety:** The compiler verifies that your code handles every possible message type. This prevents unexpected errors at runtime if a message type is overlooked. -* **High Performance:** This design allows the compiler to generate highly optimized code for processing different messages, which is typically faster than conventional virtual function lookups. -* **Efficient Memory Layout:** Messages are stored contiguously in memory, which improves CPU cache utilization and overall processing speed. +- **Compile-Time Safety:** The compiler verifies that your code handles every possible message type. This prevents unexpected errors at runtime if a message type is overlooked. +- **High Performance:** This design allows the compiler to generate highly optimized code for processing different messages, which is typically faster than conventional virtual function lookups. +- **Efficient Memory Layout:** Messages are stored contiguously in memory, which improves CPU cache utilization and overall processing speed. ### Endianness and Byte Order @@ -104,12 +131,13 @@ The parser handles this discrepancy automatically. After a message is read into ### Prerequisites -* **C++ Compiler**: A compiler with full C++20 support (e.g., GCC 10+, Clang 12+, MSVC 19.29+). -* **CMake**: Version 3.20 or newer. +- **C++ Compiler**: A compiler with full C++20 support (e.g., GCC 10+, Clang 12+, MSVC 19.29+). +- **CMake**: Version 3.25 or newer. ### Building the Project 1. **Clone the Repository** + ```bash git clone https://github.com/bbalouki/itchcpp.git cd itchcpp @@ -117,6 +145,7 @@ The parser handles this discrepancy automatically. After a message is read into 2. **Configure with CMake** This step generates the build system and automatically downloads any developer dependencies if needed. + ```bash cmake -S . -B build ``` @@ -131,29 +160,31 @@ The parser handles this discrepancy automatically. After a message is read into The project includes unit tests and performance benchmarks that can be built by enabling the corresponding CMake options. -* **To build and run tests:** - ```bash - # Configure with tests enabled - cmake -S . -B build -DITCH_BUILD_TESTS=ON +- **To build and run tests:** - # Build - cmake --build build --config Release + ```bash + # Configure with tests enabled + cmake -S . -B build -DITCH_BUILD_TESTS=ON - # Run tests from the build directory - ctest --test-dir build --output-on-failure - ``` + # Build + cmake --build build --config Release -* **To build and run benchmarks:** - ```bash - # Configure with benchmarks enabled - cmake -S . -B build -DITCH_BUILD_BENCHMARKS=ON + # Run tests from the build directory + ctest --test-dir build --output-on-failure + ``` - # Build - cmake --build build --config Release +- **To build and run benchmarks:** - # Run benchmarks - ./build/benchmarks/parser_bench - ``` + ```bash + # Configure with benchmarks enabled + cmake -S . -B build -DITCH_BUILD_BENCHMARKS=ON + + # Build + cmake --build build --config Release + + # Run benchmarks + ./build/benchmarks/parser_bench + ``` --- @@ -162,8 +193,9 @@ The project includes unit tests and performance benchmarks that can be built by This library has **zero external runtime dependencies**. For development purposes, it uses two popular libraries: -* **Google Test**: For unit testing. -* **Google Benchmark**: For performance microbenchmarks. + +- **Google Test**: For unit testing. +- **Google Benchmark**: For performance microbenchmarks. These are handled automatically by Microsoft's `vcpkg` package manager. When you enable tests or benchmarks, **You will need to install them on your system separately or install them in manifest mode.** @@ -311,20 +343,20 @@ This standalone example demonstrates how to use `std::visit` with a visitor to p // Using `if constexpr` ensures only valid code is compiled for each message type. auto message_visitor = [](const auto& msg) { using T = std::decay_t; - + // Set fixed-point notation for prices std::cout << std::fixed << std::setprecision(4); if constexpr (std::is_same_v) { std::cout << "ADD ORDER: Ref #" << msg.order_reference_number - << ", " << msg.shares << " shares of " + << ", " << msg.shares << " shares of " << itch::to_string(msg.stock, sizeof(msg.stock)) // Helper to convert char array << " @ $" << static_cast(msg.price) / itch::PRICE_DIVISOR << "\n"; } else if constexpr (std::is_same_v) { std::cout << "EXECUTION: Ref #" << msg.order_reference_number << ", executed " << msg.executed_shares << " shares.\n"; } else if constexpr (std::is_same_v) { - std::cout << "SYSTEM EVENT: Code '" << msg.event_code + std::cout << "SYSTEM EVENT: Code '" << msg.event_code << "' (O=Start, S=SysHours, Q=MktHours, M=EndMkt, E=EndSys, C=End)\n"; } else { // This block will be visited for any other message types. @@ -388,7 +420,7 @@ int main(int argc, char* argv[]) { // Create a parser and configure it to only process book-related messages // This is a performance optimization. itch::Parser parser; - std::vector book_messages(order_book.book_messages.begin(), + std::vector book_messages(order_book.book_messages.begin(), order_book.book_messages.end()); try { @@ -421,9 +453,9 @@ The `print()` method provides a formatted snapshot of the top of the book. It di +-----------------------------------------------------+ | AAPL | +--------------------------+--------------------------+ -| ASKS (SELL) | BIDS (BUY) | +| ASKS (SELL) | BIDS (BUY) | +------------+-------------+------------+-------------+ -| PRICE | VOLUME | PRICE | VOLUME | +| PRICE | VOLUME | PRICE | VOLUME | +------------+-------------+------------+-------------+ | 173.5000 | 15,000 | 173.4900 | 23,500 | | 173.5100 | 12,300 | 173.4800 | 18,900 | @@ -433,6 +465,184 @@ The `print()` method provides a formatted snapshot of the top of the book. It di +------------+-------------+------------+-------------+ ``` +### Feed Ingestion (Transport) + +The raw parser assumes a concatenation of length-prefixed messages, but ITCH is +delivered inside transport framing. The `itch::transport` module decodes the +framing that actually arrives on the wire and on disk, then feeds the existing +parser. Everything is implemented in-house with **no libpcap dependency**. + +| Decoder | Header | Purpose | +| ------------------ | ------------------------------- | ------------------------------------------------------------- | +| `MoldUdp64Decoder` | `itch/transport/moldudp64.hpp` | UDP multicast framing for live dissemination. | +| `SoupBinDecoder` | `itch/transport/soupbintcp.hpp` | TCP framing for Glimpse snapshots and recovery/replay. | +| `PcapReader` | `itch/transport/pcap.hpp` | Replay a feed from a `.pcap`/`.pcapng` capture file. | +| `SequenceTracker` | `itch/transport/sequencing.hpp` | Per-session sequence tracking, gap detection, recovery hooks. | + +```cpp +#include "itch/transport/pcap.hpp" + +itch::transport::PcapReader reader{[](const itch::Message& msg) { + // ... handle each decoded ITCH message ... +}}; + +// Surface any sequence gaps in the multicast stream. +reader.mold_decoder().tracker().set_gap_callback( + [](std::string_view session, std::uint64_t expected, std::uint64_t got) { + // ... drive recovery / log the gap ... + }); + +reader.read_file("capture.pcapng"); // walks Ethernet/IP/UDP/MoldUDP64 +// reader.messages_decoded(), reader.udp_datagrams(), tracker().gap_count() +``` + +For a live or captured MoldUDP64 datagram you already have in memory, call +`MoldUdp64Decoder::decode_packet(span)` directly; for a SoupBinTCP byte stream, +push segments through `SoupBinDecoder::feed(span)` as they arrive. + +### Full-Market Book Engine + +The original `LimitOrderBook` reconstructs a single, pre-selected symbol. The +Phase 2 `itch::book::BookManager` reconstructs **every** symbol on the feed in one +pass, routing each message to that security's book by stock locate code in O(1). +Each book (`itch::book::L3Book`) is allocation-light: orders live in a reusable +object pool linked into intrusive FIFO queues, price levels are kept in flat +sorted ladders, and order lookup by reference number uses a flat open-addressed +map, so there is no per-order heap allocation or atomic refcount on the hot path. + +```cpp +#include "itch/book/book_manager.hpp" +#include "itch/parser.hpp" + +itch::book::BookManager manager; +// manager.track_symbol("AAPL"); // optional: restrict to a universe + +manager.set_bbo_callback([](const itch::book::L3Book& book, const itch::book::Bbo& bbo) { + // best bid/offer for book.symbol() just changed +}); +manager.set_trade_callback([](const itch::Trade& trade) { + // trade.printable distinguishes displayable prints from hidden ones +}); + +itch::Parser parser; +parser.parse(buffer.data(), buffer.size(), [&](const itch::Message& msg) { + manager.process(msg); +}); + +const itch::book::L3Book* book = manager.book_for_symbol("AAPL"); +auto top = book->bbo(); // best bid/offer +auto l2 = book->depth(itch::book::Side::buy, 5); // top-5 aggregated bids +auto l3 = book->orders_at(itch::book::Side::buy, top.bid_price.raw()); // order-level +``` + +For callers that touch only a few fields per message, `itch/overlay.hpp` provides +a zero-copy alternative to the eager parser: `for_each_message(buffer, cb)` yields +a `MessageView` (and typed views like `AddOrderView`) that decode each field +lazily on access. + +### Analytics + +The header-only `itch::analytics` layer computes the metrics quants ask for, +directly off the trade tape and book so every downstream team does not +reimplement them. + +| Component | Header | Provides | +| ------------------------ | ------------------------------ | ------------------------------------------------------------------- | +| `BarBuilder` | `analytics/bars.hpp` | OHLCV bars over `TimeClock`, `TickClock`, `VolumeClock`. | +| `Vwap`, `Twap` | `analytics/vwap.hpp` | Running/interval volume- and time-weighted average price. | +| spread / mid / imbalance | `analytics/microstructure.hpp` | Spread, mid, depth-at-level, queue imbalance, order-flow imbalance. | +| `ImbalanceInfo` | `analytics/imbalance.hpp` | Decoded NOII (`I`) imbalance data. | +| `AuctionTracker` | `analytics/auctions.hpp` | Opening/closing/halt/IPO cross reconstruction. | + +```cpp +#include "itch/analytics/vwap.hpp" +#include "itch/analytics/bars.hpp" + +itch::analytics::Vwap vwap; +itch::analytics::BarBuilder bars{itch::analytics::TimeClock{60'000'000'000ULL}, // 1-minute bars + [](const itch::analytics::Bar& bar) { /* ... */ }}; + +manager.set_trade_callback([&](const itch::Trade& trade) { + vwap.add(trade.price, trade.shares); + bars.add(trade); +}); +// ... parse the feed ... +bars.flush(); +double session_vwap = vwap.value(); +``` + +### Interoperability + +Meet researchers in the tools they already use. + +**Output sinks.** `itch/io/csv_sink.hpp` flattens any message stream into a wide +CSV table (dependency-free). With `-DITCH_WITH_ARROW=ON` (the `arrow` vcpkg +feature), `itch/io/arrow_export.hpp` writes Parquet for pandas/Polars/DuckDB/Spark. + +**`itch-tool` CLI** (`-DITCH_BUILD_TOOLS=ON`). Inspect, filter, and convert feeds +without writing code; input may be a raw ITCH stream or a `.pcap`/`.pcapng` +capture (auto-detected): + +```bash +itch-tool stats data.itch # per-type message histogram +itch-tool inspect data.pcapng --limit 50 # human-readable dump +itch-tool filter data.itch --types AEP --out trades.csv +itch-tool convert data.itch --out data.csv # ITCH -> CSV (-> Parquet w/ Arrow) +``` + +**Python bindings** (`-DITCH_BUILD_PYTHON=ON`, pybind11). The `itchcpp` package is +a faster, drop-in backend for the pure-Python +[`itch`](https://github.com/bbalouki/itch) package (PyPI: +[`itchfeed`](https://pypi.org/project/itchfeed/)). It mirrors that package's layout +(`itchcpp.messages`, `itchcpp.parser`, `itchcpp.indicators`) and semantics, the same +message classes with the same raw `bytes`/`int` attributes, the same `MessageParser` +(type filter, lazy iteration, `parse_file`/`parse_stream`/`parse_messages`), +`create_message`, and `decode`/`decode_price`/`to_bytes` helpers, so migrating only +changes the import root. See [itchcpp](python/README.md). + +```python +from itchcpp.parser import MessageParser +from itchcpp.messages import AddOrderNoMPIAttributionMessage + +parser = MessageParser() # MessageParser(message_type=b"AFE") to filter types +with open("01302020.NASDAQ_ITCH50", "rb") as itch_file: + for message in parser.parse_file(itch_file): + if isinstance(message, AddOrderNoMPIAttributionMessage): + print(message.stock, message.decode_price("price"), message.shares) +``` + +### Simulation & Ecosystem + +**Replay engine** (`itch/replay.hpp`). Drive a consumer at the feed's original +cadence (or a scaled speed) for realistic backtesting: + +```cpp +#include "itch/replay.hpp" + +itch::ReplayEngine engine{10.0}; // 10x real time; <= 0 means as fast as possible +engine.replay(buffer, [](const itch::Message& msg) { /* paced by timestamps */ }); +``` + +**Encoder / writer** (`itch/encoder.hpp`). Serialize any message back to valid +wire bytes, with a guaranteed `parse(encode(msg)) == msg` round-trip, used to +synthesize streams and golden fixtures: + +```cpp +#include "itch/encoder.hpp" + +std::vector frame = itch::encode_frame(message); // length-prefixed +``` + +**Multi-venue seam** (`itch/venue.hpp`). NASDAQ TotalView-ITCH 5.0 is the only +implemented venue; `itch::venue::VenuePolicy` (modelled by `itch::venue::Nasdaq50`) +is the extension point for adding BX/PSX or older ITCH versions without rewriting +the dispatch machinery. + +**Packaging.** The library is consumable through vcpkg (manifest with optional +`python` and `arrow` features) and Conan (`conanfile.py`). See +[CONTRIBUTING](CONTRIBUTING.md) for the build options and the versioning/ABI +compatibility policy. + --- ## API Reference: ITCH 5.0 Message Types @@ -441,54 +651,51 @@ This library supports all message types defined in the ITCH 5.0 specification. T ### System Event Messages -| Type | Struct Name | Description | -| :--: | -------------------- | ---------------------------------------------- | -| `S` | `SystemEventMessage` | Signals a market or data feed handler event. | +| Type | Struct Name | Description | +| :--: | -------------------- | -------------------------------------------- | +| `S` | `SystemEventMessage` | Signals a market or data feed handler event. | ### Stock and Administrative Messages -| Type | Struct Name | Description | -| :--: | ---------------------------------- | ------------------------------------------------------------------------- | -| `R` | `StockDirectoryMessage` | Conveys stock symbol directory information. | -| `H` | `StockTradingActionMessage` | Indicates a change in trading status for a security (e.g., Halted, Trading).| -| `Y` | `RegSHORestrictionMessage` | Regulation SHO short sale price test restricted indicator. | -| `L` | `MarketParticipantPositionMessage` | Conveys a market participant's status (e.g., Primary Market Maker). | -| `V` | `MWCBDeclineLevelMessage` | Informs of the Market-Wide Circuit Breaker (MWCB) breach points for the day.| -| `W` | `MWCBStatusMessage` | Indicates that a MWCB level has been breached. | -| `h` | `OperationalHaltMessage` | Indicates an operational halt for a security on a specific market center. | - +| Type | Struct Name | Description | +| :--: | ---------------------------------- | ---------------------------------------------------------------------------- | +| `R` | `StockDirectoryMessage` | Conveys stock symbol directory information. | +| `H` | `StockTradingActionMessage` | Indicates a change in trading status for a security (e.g., Halted, Trading). | +| `Y` | `RegSHORestrictionMessage` | Regulation SHO short sale price test restricted indicator. | +| `L` | `MarketParticipantPositionMessage` | Conveys a market participant's status (e.g., Primary Market Maker). | +| `V` | `MWCBDeclineLevelMessage` | Informs of the Market-Wide Circuit Breaker (MWCB) breach points for the day. | +| `W` | `MWCBStatusMessage` | Indicates that a MWCB level has been breached. | +| `h` | `OperationalHaltMessage` | Indicates an operational halt for a security on a specific market center. | ### Order Messages -| Type | Struct Name | Description | -| :--: | --------------------------------- | ---------------------------------------------------------------------- | -| `A` | `AddOrderMessage` | A new order has been accepted (no MPID attribution). | -| `F` | `AddOrderMPIDMessage` | A new order has been accepted (with MPID attribution). | -| `E` | `OrderExecutedMessage` | An order on the book was executed in part or in full. | -| `C` | `OrderExecutedWithPriceMessage` | An order was executed at a price different from the display price. | -| `X` | `OrderCancelMessage` | An order was partially canceled; this message contains the canceled quantity. | -| `D` | `OrderDeleteMessage` | An order was canceled in its entirety and removed from the book. | -| `U` | `OrderReplaceMessage` | An existing order has been replaced with a new order. | - +| Type | Struct Name | Description | +| :--: | ------------------------------- | ----------------------------------------------------------------------------- | +| `A` | `AddOrderMessage` | A new order has been accepted (no MPID attribution). | +| `F` | `AddOrderMPIDMessage` | A new order has been accepted (with MPID attribution). | +| `E` | `OrderExecutedMessage` | An order on the book was executed in part or in full. | +| `C` | `OrderExecutedWithPriceMessage` | An order was executed at a price different from the display price. | +| `X` | `OrderCancelMessage` | An order was partially canceled; this message contains the canceled quantity. | +| `D` | `OrderDeleteMessage` | An order was canceled in its entirety and removed from the book. | +| `U` | `OrderReplaceMessage` | An existing order has been replaced with a new order. | ### Trade Messages -| Type | Struct Name | Description | -| :--: | -------------------- | ----------------------------------------------------------------------------- | -| `P` | `TradeMessage` | Reports a trade for a non-displayable order type. | -| `Q` | `CrossTradeMessage` | Reports a cross trade (e.g., Opening, Closing, IPO Cross). | -| `B` | `BrokenTradeMessage` | Reports that a previously disseminated execution has been broken. | - +| Type | Struct Name | Description | +| :--: | -------------------- | ----------------------------------------------------------------- | +| `P` | `TradeMessage` | Reports a trade for a non-displayable order type. | +| `Q` | `CrossTradeMessage` | Reports a cross trade (e.g., Opening, Closing, IPO Cross). | +| `B` | `BrokenTradeMessage` | Reports that a previously disseminated execution has been broken. | ### Imbalance and Price Discovery Messages -| Type | Struct Name | Description | -| :--: | -------------------------------------------- | ---------------------------------------------------------------------------- | -| `K` | `IPOQuotingPeriodUpdateMessage` | Provides the anticipated IPO quotation release time. | -| `J` | `LULDAuctionCollarMessage` | Indicates auction collar thresholds for a security in a LULD Trading Pause. | -| `I` | `NOIIMessage` | Net Order Imbalance Indicator message, used for opening/closing crosses. | -| `N` | `RetailPriceImprovementIndicator` | Indicates the presence of Retail Price Improvement (RPII) interest. | -| `O` | `DirectListingCapitalRaisePriceDiscovery` | Disseminated for Direct Listing with Capital Raise (DLCR) securities. | +| Type | Struct Name | Description | +| :--: | ----------------------------------------- | --------------------------------------------------------------------------- | +| `K` | `IPOQuotingPeriodUpdateMessage` | Provides the anticipated IPO quotation release time. | +| `J` | `LULDAuctionCollarMessage` | Indicates auction collar thresholds for a security in a LULD Trading Pause. | +| `I` | `NOIIMessage` | Net Order Imbalance Indicator message, used for opening/closing crosses. | +| `N` | `RetailPriceImprovementIndicator` | Indicates the presence of Retail Price Improvement (RPII) interest. | +| `O` | `DirectListingCapitalRaisePriceDiscovery` | Disseminated for Direct Listing with Capital Raise (DLCR) securities. | --- @@ -496,16 +703,55 @@ This library supports all message types defined in the ITCH 5.0 specification. T The parser is designed for high-throughput scenarios. Performance is heavily dependent on the underlying hardware (CPU speed, memory bandwidth, and I/O speed). -To run the performance benchmarks: +### Benchmarks + +The numbers below were produced by [`benchmarks/parser_bench.cpp`](benchmarks/parser_bench.cpp) parsing from an in-memory buffer (file I/O excluded). They are reproducible with the setup described under [How to reproduce](#how-to-reproduce). + +| Configuration | Throughput | +| ------------------------------------------------------------------ | -------------- | +| `parse(..., callback)`, allocation-free callback path | **2.24 GiB/s** | +| `parse(...)`, collect all messages into a `std::vector` | 1.23 GiB/s | +| `parse(..., {'A','P','E','C','X'})`, filtered into a `std::vector` | 1.15 GiB/s | + +- **Hardware**: x86-64, 16 logical cores @ 1.70 GHz (L1d 32 KiB, L2 512 KiB, L3 4 MiB). +- **Compiler**: Clang 22, `-DCMAKE_BUILD_TYPE=Release`, C++23. +- **Dataset**: official NASDAQ TotalView-ITCH 5.0 sample (`01302020`), a ~300 MB frame-aligned slice loaded fully into memory. + +The collect/filter paths are slower than the callback path because they allocate and copy each parsed message into a `std::vector`; the callback path does neither, which is why it is the recommended interface for latency-sensitive processing. + +#### Book engine and overlay + +These are produced by [`benchmarks/book_bench.cpp`](benchmarks/book_bench.cpp). +The throughput figures below were measured on a synthetic, churn-heavy stream +(adds and deletes around a moving mid across 100 symbols) on the same hardware, +Clang 22 `-DCMAKE_BUILD_TYPE=Release`, C++23; book-rebuild throughput on real +data depends heavily on the depth and churn profile of the feed. + +| Configuration | Throughput | +| --------------------------------------------------------------------- | -------------- | +| `BookManager::process`, full multi-symbol L3 reconstruction | ~150 MiB/s | +| Eager `parse`, touching one field per message | ~5.4 GiB/s | +| Zero-copy `overlay::for_each_message`, touching one field per message | **~9.7 GiB/s** | + +The overlay is materially faster than the eager decoder when only a few fields are +read, because it never decodes the fields the caller does not touch. Reproduce +with `./build/benchmarks/book_bench `. + +### How to reproduce + ```bash -# Configure and build with benchmarks enabled -cmake -S . -B build -DITCH_BUILD_BENCHMARKS=ON +# 1. Fetch an official sample (multi-GB; not committed to the repo). +scripts/fetch_sample.sh data + +# 2. Configure and build with benchmarks enabled. +cmake -S . -B build -DITCH_BUILD_BENCHMARKS=ON -DITCH_CXX_STANDARD=23 -DCMAKE_BUILD_TYPE=Release cmake --build build --config Release -# Run the benchmark suite -./build/benchmarks/parser_bench +# 3. Run the benchmark suite against the sample. +./build/benchmarks/parser_bench data/01302020.NASDAQ_ITCH50 ``` -The output will show the parsing rate in messages per second and throughput in GB/s. On modern server hardware, expect throughput in the multi-gigabytes per second range when parsing from an in-memory buffer. + +The output reports throughput in GiB/s for each parsing strategy. --- @@ -516,14 +762,15 @@ The project uses `.clang-format` to enforce a consistent code style. A `.clang-t --- ## Frequently Asked Questions (FAQ) + - **What exactly is this NASDAQ ITCH data?** -It's the most detailed data feed available from NASDAQ. It's not just stock prices; it’s every single order placed, modified, cancelled, and executed. It's the complete "play-by-play" of the market. + It's the most detailed data feed available from NASDAQ. It's not just stock prices; it’s every single order placed, modified, cancelled, and executed. It's the complete "play-by-play" of the market. - **Why can't I just use a simple program to read this data?** -The data is in a highly optimized, machine-only binary format, not human-readable text. More importantly, the volume and velocity are immense—a single day of trading can generate tens or hundreds of gigabytes of data. A standard program would be far too slow to keep up. + The data is in a highly optimized, machine-only binary format, not human-readable text. More importantly, the volume and velocity are immense—a single day of trading can generate tens or hundreds of gigabytes of data. A standard program would be far too slow to keep up. - **Is this a program I can just double-click and run?** -No, it’s a specialized component—a library—that software developers use as the "engine" inside a larger application (like a trading platform or an analysis tool). It does the heavy lifting of data processing so they can focus on building the features their users need. + No, it’s a specialized component—a library—that software developers use as the "engine" inside a larger application (like a trading platform or an analysis tool). It does the heavy lifting of data processing so they can focus on building the features their users need. --- @@ -537,4 +784,4 @@ This project is licensed under the MIT License. See the [LICENSE](LICENSE) file ## References -* **Nasdaq TotalView-ITCH 5.0 Specification:** The official [documentation](https://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NQTVITCHspecification.pdf) is the definitive source for protocol details. +- **Nasdaq TotalView-ITCH 5.0 Specification:** The official [documentation](https://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NQTVITCHspecification.pdf) is the definitive source for protocol details. diff --git a/VERSION.txt b/VERSION.txt index 653f458..dc1e644 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1,2 +1 @@ -1.1.0 - +1.6.0 diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 2007ce9..778c432 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.23) +cmake_minimum_required(VERSION 3.25) find_package(benchmark CONFIG QUIET) @@ -15,7 +15,15 @@ endif() add_executable(parser_bench parser_bench.cpp) target_link_libraries( - parser_bench PRIVATE + parser_bench PRIVATE + itch::itch + benchmark::benchmark + benchmark::benchmark_main +) + +add_executable(book_bench book_bench.cpp) +target_link_libraries( + book_bench PRIVATE itch::itch benchmark::benchmark benchmark::benchmark_main diff --git a/benchmarks/book_bench.cpp b/benchmarks/book_bench.cpp new file mode 100644 index 0000000..10fbe08 --- /dev/null +++ b/benchmarks/book_bench.cpp @@ -0,0 +1,125 @@ +/// @file book_bench.cpp +/// @brief Benchmarks for the Phase 2 book engine and the zero-copy overlay. +/// +/// Measures three things against an in-memory ITCH buffer (file I/O excluded): +/// - BM_BookRebuild: full multi-symbol book reconstruction through BookManager. +/// - BM_EagerTouch: eager parse touching one field per message (the baseline). +/// - BM_OverlayTouch: lazy overlay framing touching the same one field, which +/// should be cheaper because the other fields are never decoded. +/// +/// Usage: +/// ./book_bench [google benchmark options] + +#include + +#include +#include +#include +#include +#include + +#include "itch/book/book_manager.hpp" +#include "itch/overlay.hpp" +#include "itch/parser.hpp" + +namespace data { +// NOLINTNEXTLINE +std::string g_data_filename {}; +} // namespace data + +namespace { + +auto load_file(const std::string& path) -> std::vector { + std::ifstream file {path, std::ios::binary}; + file.seekg(0, std::ios::end); + const auto size = file.tellg(); + file.seekg(0, std::ios::beg); + std::vector buffer(static_cast(size)); + file.read(reinterpret_cast(buffer.data()), size); // NOLINT + return buffer; +} + +class BookBenchmark : public benchmark::Fixture { + public: + std::vector itch_data; + + void SetUp(::benchmark::State& state) override { + if (data::g_data_filename.empty()) { + state.SkipWithError("ITCH data file not provided."); + return; + } + itch_data = load_file(data::g_data_filename); + if (itch_data.empty()) { + state.SkipWithError("Failed to read ITCH data file."); + } + } + void TearDown(const ::benchmark::State&) override { + itch_data.clear(); + itch_data.shrink_to_fit(); + } +}; + +} // namespace + +BENCHMARK_F(BookBenchmark, BM_BookRebuild)(benchmark::State& state) { + std::size_t total_bytes = 0; + for ([[maybe_unused]] auto iter : state) { + itch::Parser parser; + itch::book::BookManager manager; + parser.parse(std::span {itch_data}, [&](const itch::Message& msg) { + manager.process(msg); + }); + benchmark::DoNotOptimize(manager.book_count()); + total_bytes += itch_data.size(); + } + state.SetBytesProcessed(static_cast(total_bytes)); +} + +BENCHMARK_F(BookBenchmark, BM_EagerTouch)(benchmark::State& state) { + std::size_t total_bytes = 0; + for ([[maybe_unused]] auto iter : state) { + itch::Parser parser; + std::uint64_t sink = 0; + parser.parse(std::span {itch_data}, [&](const itch::Message& msg) { + sink += std::visit( + [](const auto& inner) -> std::uint64_t { return inner.stock_locate; }, msg + ); + }); + benchmark::DoNotOptimize(sink); + total_bytes += itch_data.size(); + } + state.SetBytesProcessed(static_cast(total_bytes)); +} + +BENCHMARK_F(BookBenchmark, BM_OverlayTouch)(benchmark::State& state) { + std::size_t total_bytes = 0; + for ([[maybe_unused]] auto iter : state) { + std::uint64_t sink = 0; + itch::overlay::for_each_message( + std::span {itch_data}, + [&](const itch::overlay::MessageView& view) { sink += view.stock_locate(); } + ); + benchmark::DoNotOptimize(sink); + total_bytes += itch_data.size(); + } + state.SetBytesProcessed(static_cast(total_bytes)); +} + +auto main(int argc, char** argv) -> int { + if (argc < 2) { + std::cerr << "Usage: ./book_bench [benchmark options]\n"; + return 1; + } + data::g_data_filename = argv[1]; + for (int index = 1; index < argc - 1; ++index) { + argv[index] = argv[index + 1]; + } + --argc; + benchmark::Initialize(&argc, argv); + if (benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} diff --git a/benchmarks/parser_bench.cpp b/benchmarks/parser_bench.cpp index 38b61ea..5221d4e 100644 --- a/benchmarks/parser_bench.cpp +++ b/benchmarks/parser_bench.cpp @@ -1,27 +1,26 @@ -/** - * @file parser_bench.cpp - * @brief Benchmarking suite for the ITCH message parser using Google Benchmark. - * - * This file defines a set of benchmarks to evaluate the performance of the - * ITCH message parser implemented in the Parser class. It uses the Google - * Benchmark library to measure execution time and throughput for different - * parsing strategies, including callback-based parsing, collecting all parsed - * messages, and filtering specific message types. - * - * The benchmark fixture `ParserBenchmark` is responsible for loading the ITCH - * data file into memory once per benchmark run, ensuring that file I/O does - * not skew the parsing performance measurements. - * - * Usage: - * ./parser_bench.exe [google benchmark options] - * - * Example: - * ./parser_bench.exe data/itch_data.bin --benchmark_filter=BM_ParseWithCallback - * - * Note: - * Ensure that the Google Benchmark library is properly linked during - * compilation. - */ + +/// @file parser_bench.cpp +/// @brief Benchmarking suite for the ITCH message parser using Google Benchmark. +/// +/// This file defines a set of benchmarks to evaluate the performance of the +/// ITCH message parser implemented in the Parser class. It uses the Google +/// Benchmark library to measure execution time and throughput for different +/// parsing strategies, including callback-based parsing, collecting all parsed +/// messages, and filtering specific message types. +/// +/// The benchmark fixture `ParserBenchmark` is responsible for loading the ITCH +/// data file into memory once per benchmark run, ensuring that file I/O does +/// not skew the parsing performance measurements. +/// +/// Usage: +/// ./parser_bench.exe [google benchmark options] +/// +/// Example: +/// ./parser_bench.exe data/itch_data.bin --benchmark_filter=BM_ParseWithCallback +/// +/// Note: +/// Ensure that the Google Benchmark library is properly linked during +/// compilation. #include diff --git a/cmake/Helpers.cmake b/cmake/Helpers.cmake index 8af1937..75e74e6 100644 --- a/cmake/Helpers.cmake +++ b/cmake/Helpers.cmake @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.23) +cmake_minimum_required(VERSION 3.25) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -8,16 +8,20 @@ set(CMAKE_CXX_STANDARD ${${PROJECT_NAME}_CXX_STANDARD}) option(${PROJECT_NAME}_BUILD_TESTS "Add tests" OFF) option(${PROJECT_NAME}_BUILD_BENCHMARKS "Add benchmark analysis" OFF) option(${PROJECT_NAME}_BUILD_EXAMPLES "Build some examples" OFF) +option(${PROJECT_NAME}_BUILD_FUZZERS "Build libFuzzer targets (Clang only)" OFF) +option(${PROJECT_NAME}_BUILD_TOOLS "Build the itch-tool command-line utility" OFF) +option(${PROJECT_NAME}_BUILD_PYTHON "Build the pybind11 Python bindings" OFF) +option(${PROJECT_NAME}_WITH_ARROW "Enable Apache Arrow / Parquet export" OFF) set(${PROJECT_NAME}_PROJECT_ENV "DEV" CACHE STRING "Development environment") set_property(CACHE ${PROJECT_NAME}_PROJECT_ENV PROPERTY STRINGS "DEV" "PROD") message(STATUS "Building for environment: ${${PROJECT_NAME}_PROJECT_ENV}") -option(${PROJECT_NAME}_ADD_COVERAGE_ANALYSIS "Enable coverage analisys" OFF) -option(${PROJECT_NAME}_APPLY_FORMATING "Apply formating with clang-format" OFF) +option(${PROJECT_NAME}_ADD_COVERAGE_ANALYSIS "Enable coverage analysis" OFF) +option(${PROJECT_NAME}_APPLY_FORMATING "Apply formatting with clang-format" OFF) option(${PROJECT_NAME}_USE_PER_FILE_FORMATTING "For very large projects" OFF) -option(${PROJECT_NAME}_APPLY_CLANG_TIDY_GLOBALY "Apply clang tidy globaly" OFF) -option(${PROJECT_NAME}_BUILD_DOCUMENTATION "Build documenation with Doxygen" OFF) +option(${PROJECT_NAME}_APPLY_CLANG_TIDY_GLOBALY "Apply clang tidy globally" OFF) +option(${PROJECT_NAME}_BUILD_DOCUMENTATION "Build documentation with Doxygen" OFF) option(${PROJECT_NAME}_ENABLE_ADDRESS_SANITIZER "Enable Address Sanitizer" OFF) ###################################################### @@ -86,7 +90,7 @@ function(apply_clang_tidy TARGET_NAME) endfunction() ################################################### -function(apply_clang_tidy_globaly) +function(apply_clang_tidy_globally) find_program(CLANG_TIDY_EXE "clang-tidy") if(CLANG_TIDY_EXE) @@ -133,14 +137,54 @@ function(add_coverage_analysis) endfunction() #################################################### -function(build_documenation) +function(build_documentation) find_package(Doxygen) if (Doxygen_FOUND) message( - STATUS - "Found Doxygen Version " "${DOXYGEN_VERSION} " + STATUS + "Found Doxygen Version " "${DOXYGEN_VERSION} " "at ${DOXYGEN_EXECUTABLE}" ) + + # Fetch the doxygen-awesome-css theme. + include(FetchContent) + FetchContent_Declare( + doxygen-awesome-css + GIT_REPOSITORY https://github.com/jothepro/doxygen-awesome-css.git + GIT_TAG "v${DOXYGEN_AWESOME_VERSION}" + SOURCE_SUBDIR "do-not-add-subdir" + ) + FetchContent_MakeAvailable(doxygen-awesome-css) + set(DOXYGEN_AWESOME_DIR "${doxygen-awesome-css_SOURCE_DIR}") + + # Generate a header from the active Doxygen version and inject the theme scripts before + # . Generating against the real binary keeps the header in sync with whatever + # Doxygen produces, instead of committing a hand-written header that drifts per version. + set(DOXYGEN_DEFAULT_HEADER "${CMAKE_CURRENT_BINARY_DIR}/header.default.html") + set(DOXYGEN_HTML_HEADER "${CMAKE_CURRENT_BINARY_DIR}/header.html") + execute_process( + COMMAND "${DOXYGEN_EXECUTABLE}" -w html + "${DOXYGEN_DEFAULT_HEADER}" + "${CMAKE_CURRENT_BINARY_DIR}/footer.default.html" + "${CMAKE_CURRENT_BINARY_DIR}/style.default.css" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + ) + file(READ "${DOXYGEN_DEFAULT_HEADER}" DOXYGEN_HEADER_CONTENT) + string(CONCAT DOXYGEN_THEME_SCRIPTS + "\n" + "\n" + "\n" + "\n" + "\n" + ) + string(REPLACE "" "${DOXYGEN_THEME_SCRIPTS}" DOXYGEN_HEADER_CONTENT "${DOXYGEN_HEADER_CONTENT}") + file(WRITE "${DOXYGEN_HTML_HEADER}" "${DOXYGEN_HEADER_CONTENT}") + configure_file( "${PROJECT_SOURCE_DIR}/docs/Doxyfile.in" "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" @@ -152,11 +196,12 @@ function(build_documenation) COMMENT "Generating docs with Doxygen ..." ) message(STATUS "Add 'docs' target") + message(STATUS "To build the documentation, run: cmake --build . --target docs") endif() endfunction() ##################################################### -function(apply_formating USE_PER_FILE_LOGIC) +function(apply_formatting USE_PER_FILE_LOGIC) find_program(CLANG_FORMAT_EXE clang-format) if (NOT CLANG_FORMAT_EXE) @@ -203,7 +248,7 @@ function(apply_formating USE_PER_FILE_LOGIC) endfunction() ##################################################### -function(anable_address_sanitizer) +function(enable_address_sanitizer) set(SANITIZER_SUPPORTED OFF) if (MINGW) if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") diff --git a/cmake/Versions.cmake b/cmake/Versions.cmake index 7222516..00c5260 100644 --- a/cmake/Versions.cmake +++ b/cmake/Versions.cmake @@ -1,4 +1,5 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) set(GTEST_VERSION 1.17.0 CACHE STRING "") set(BENCHMARK_VERSION 1.9.4 CACHE STRING "") +set(DOXYGEN_AWESOME_VERSION 2.4.2 CACHE STRING "") diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000..dd2e7fa --- /dev/null +++ b/conanfile.py @@ -0,0 +1,71 @@ +"""Conan recipe for ITCHCPP. + +Packages the header-only-friendly ITCH 5.0 library (parser, transport decoders, +book engine, analytics, encoder, replay) so downstream projects can depend on it +through Conan. Tests, benchmarks, tools, and the optional Arrow/Python features are +not part of the consumed package; enable them from a source build instead. +""" + +import os + +from conan import ConanFile +from conan.tools.cmake import CMake, CMakeToolchain, cmake_layout +from conan.tools.files import copy, load + + +class ItchConan(ConanFile): + name = "itchcpp" + license = "MIT" + url = "https://github.com/bbalouki/itchcpp" + description = ( + "High-performance C++20 NASDAQ TotalView-ITCH 5.0 parser, order-book " + "engine, analytics, and feed-ingestion library." + ) + topics = ("nasdaq", "itch", "market-data", "order-book", "hft", "finance") + settings = "os", "compiler", "build_type", "arch" + options = {"with_arrow": [True, False]} + default_options = {"with_arrow": False} + exports_sources = ( + "CMakeLists.txt", + "VERSION.txt", + "cmake/*", + "src/*", + "include/*", + ) + + def set_version(self): + self.version = load( + self, os.path.join(self.recipe_folder, "VERSION.txt") + ).strip() + + def requirements(self): + if self.options.with_arrow: + self.requires("arrow/15.0.0") + + def layout(self): + cmake_layout(self) + + def generate(self): + toolchain = CMakeToolchain(self) + toolchain.variables["ITCH_WITH_ARROW"] = self.options.with_arrow + toolchain.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def package(self): + cmake = CMake(self) + cmake.install() + copy( + self, + "LICENSE", + src=self.source_folder, + dst=os.path.join(self.package_folder, "licenses"), + ) + + def package_info(self): + self.cpp_info.libs = ["itch"] + self.cpp_info.set_property("cmake_file_name", "itch") + self.cpp_info.set_property("cmake_target_name", "itch::itch") diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in new file mode 100644 index 0000000..15f7522 --- /dev/null +++ b/docs/Doxyfile.in @@ -0,0 +1,98 @@ +# Doxygen configuration template for ITCHCPP. +# +# This is a CMake-configured template: build_documentation() in cmake/Helpers.cmake +# substitutes the @VARS@ below (the fetched doxygen-awesome-css theme directory, the +# generated HTML header, the project version, and absolute source paths) and adds a +# `docs` target. Build it with: +# +# cmake -S . -B build -DITCH_BUILD_DOCUMENTATION=ON +# cmake --build build --target docs +# +# Output is written to docs/html. The theme is fetched at configure time, so no +# theme files need to live in the repository. + +#--------------------------------------------------------------------------- +# Project +#--------------------------------------------------------------------------- +PROJECT_NAME = "ITCHCPP" +PROJECT_NUMBER = @PROJECT_VERSION@ +PROJECT_BRIEF = "High-Performance NASDAQ TotalView-ITCH 5.0 Parser & Market-Data Platform" +OUTPUT_DIRECTORY = @PROJECT_SOURCE_DIR@/docs +HTML_OUTPUT = html +CREATE_SUBDIRS = NO +FULL_PATH_NAMES = NO +JAVADOC_AUTOBRIEF = YES +QT_AUTOBRIEF = YES +TAB_SIZE = 4 +MARKDOWN_SUPPORT = YES +# GitHub-style heading anchors so the README's table-of-contents links resolve. +MARKDOWN_ID_STYLE = GITHUB +TOC_INCLUDE_HEADINGS = 4 +AUTOLINK_SUPPORT = YES +BUILTIN_STL_SUPPORT = YES + +#--------------------------------------------------------------------------- +# Build / extraction +#--------------------------------------------------------------------------- +EXTRACT_ALL = YES +EXTRACT_STATIC = YES +EXTRACT_LOCAL_CLASSES = YES +RECURSIVE = YES +SORT_MEMBER_DOCS = NO +QUIET = YES +WARN_IF_UNDOCUMENTED = NO + +#--------------------------------------------------------------------------- +# Input (absolute paths so the build works from any directory) +#--------------------------------------------------------------------------- +INPUT = @PROJECT_SOURCE_DIR@/README.md \ + @PROJECT_SOURCE_DIR@/python/README.md \ + @PROJECT_SOURCE_DIR@/include \ + @PROJECT_SOURCE_DIR@/src +FILE_PATTERNS = *.hpp *.h *.cpp *.md +USE_MDFILE_AS_MAINPAGE = @PROJECT_SOURCE_DIR@/README.md +EXCLUDE_PATTERNS = */build/* */build-*/* */.venv/* + +#--------------------------------------------------------------------------- +# Source browsing +#--------------------------------------------------------------------------- +SOURCE_BROWSER = YES +REFERENCED_BY_RELATION = YES +REFERENCES_RELATION = YES + +#--------------------------------------------------------------------------- +# Output formats +#--------------------------------------------------------------------------- +GENERATE_HTML = YES +GENERATE_LATEX = NO +GENERATE_XML = NO + +#--------------------------------------------------------------------------- +# HTML appearance - modern theme (doxygen-awesome-css) +#--------------------------------------------------------------------------- +# The theme directory (@DOXYGEN_AWESOME_DIR@) and the generated header +# (@DOXYGEN_HTML_HEADER@) are produced by build_documentation() at configure time. +GENERATE_TREEVIEW = YES +DISABLE_INDEX = NO +FULL_SIDEBAR = NO +HTML_COLORSTYLE = LIGHT +HTML_DYNAMIC_SECTIONS = YES +HTML_COPY_CLIPBOARD = NO +HTML_HEADER = @DOXYGEN_HTML_HEADER@ +HTML_EXTRA_STYLESHEET = @DOXYGEN_AWESOME_DIR@/doxygen-awesome.css \ + @DOXYGEN_AWESOME_DIR@/doxygen-awesome-sidebar-only.css \ + @DOXYGEN_AWESOME_DIR@/doxygen-awesome-sidebar-only-darkmode-toggle.css +HTML_EXTRA_FILES = @DOXYGEN_AWESOME_DIR@/doxygen-awesome-darkmode-toggle.js \ + @DOXYGEN_AWESOME_DIR@/doxygen-awesome-fragment-copy-button.js \ + @DOXYGEN_AWESOME_DIR@/doxygen-awesome-paragraph-link.js \ + @DOXYGEN_AWESOME_DIR@/doxygen-awesome-interactive-toc.js + +#--------------------------------------------------------------------------- +# Diagrams (graphviz) +#--------------------------------------------------------------------------- +HAVE_DOT = YES +DOT_IMAGE_FORMAT = svg +DOT_TRANSPARENT = YES +CLASS_GRAPH = YES +COLLABORATION_GRAPH = NO +UML_LOOK = YES diff --git a/docs/coverage.html b/docs/coverage.html deleted file mode 100644 index 8d74323..0000000 --- a/docs/coverage.html +++ /dev/null @@ -1,793 +0,0 @@ - - - - - - GCC Code Coverage Report - - - - - - -
-

GCC Code Coverage Report

-
-
-
- - - - - - - - - - - - - -
Directory:src/
Date:2026-01-17 09:41:03
Coverage: - low: ≥ 0% - medium: ≥ 75.0% - high: ≥ 90.0% -
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CoverageExecExclTotal
Lines:91.2%6920759
Functions:88.3%1730196
Branches:53.8%3240602
-
-
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FileLinesFunctionsBranches
- messages.cpp - - 96.7 - 96.7%146 / 0 / 15197.9%47 / 0 / 4850.0%103 / 0 / 206
- order_book.cpp - - 88.5 - 88.5%184 / 0 / 208100.0%36 / 0 / 3659.7%123 / 0 / 206
- parser.cpp - - 90.5 - 90.5%362 / 0 / 40080.4%90 / 0 / 11251.6%98 / 0 / 190
-
-
- - - diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index bd554a8..45e02b5 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) add_executable(parser_example parser/parser_example.cpp) @@ -15,3 +15,117 @@ target_link_libraries( itch::itch warnings::strict ) + +add_executable(pcap_replay_example transport/pcap_replay_example.cpp) + +target_link_libraries( + pcap_replay_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(book_engine_example book/book_engine_example.cpp) + +target_link_libraries( + book_engine_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(vwap_example analytics/vwap_example.cpp) + +target_link_libraries( + vwap_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(soupbintcp_example transport/soupbintcp_example.cpp) + +target_link_libraries( + soupbintcp_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(moldudp64_example transport/moldudp64_example.cpp) + +target_link_libraries( + moldudp64_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(overlay_example overlay/overlay_example.cpp) + +target_link_libraries( + overlay_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(bars_example analytics/bars_example.cpp) + +target_link_libraries( + bars_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(microstructure_example analytics/microstructure_example.cpp) + +target_link_libraries( + microstructure_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(auctions_example analytics/auctions_example.cpp) + +target_link_libraries( + auctions_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(csv_sink_example io/csv_sink_example.cpp) + +target_link_libraries( + csv_sink_example PRIVATE + itch::itch + warnings::strict +) + +if(ITCH_WITH_ARROW) + add_executable(arrow_export_example io/arrow_export_example.cpp) + + target_link_libraries( + arrow_export_example PRIVATE + itch::itch + warnings::strict + ) +endif() + +add_executable(encoder_example encoder/encoder_example.cpp) + +target_link_libraries( + encoder_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(replay_example replay/replay_example.cpp) + +target_link_libraries( + replay_example PRIVATE + itch::itch + warnings::strict +) + +add_executable(price_time_indicators_example utilities/price_time_indicators_example.cpp) + +target_link_libraries( + price_time_indicators_example PRIVATE + itch::itch + warnings::strict +) diff --git a/examples/analytics/auctions_example.cpp b/examples/analytics/auctions_example.cpp new file mode 100644 index 0000000..370fbe4 --- /dev/null +++ b/examples/analytics/auctions_example.cpp @@ -0,0 +1,96 @@ +#include +#include +#include +#include +#include + +#include "itch/analytics/auctions.hpp" +#include "itch/analytics/imbalance.hpp" +#include "itch/encoder.hpp" +#include "itch/parser.hpp" + +// Reconstructs an opening auction from the NOII imbalance messages leading up +// to a cross with `AuctionTracker`, alongside decoding a standalone NOII +// message directly with `make_imbalance_info`. +auto main() -> int { + constexpr std::uint64_t base_timestamp {34'200'000'000'000ULL}; // 09:30:00.000 + + const itch::NOIIMessage first_noii { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp, + .paired_shares = 50'000, + .imbalance_shares = 5'000, + .imbalance_direction = 'B', + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .far_price = 1'499'500, + .near_price = 1'500'000, + .current_reference_price = 1'500'000, + .cross_type = 'O', + .price_variation_indicator = 'L' + }; + + // Demonstrate decoding a single NOII message directly. + const itch::analytics::ImbalanceInfo decoded = itch::analytics::make_imbalance_info(first_noii); + std::cout << std::format( + "NOII for {}: {} shares imbalance ({}), reference price {:.4f}\n", + decoded.stock, + decoded.imbalance_shares, + itch::analytics::imbalance_direction_name(decoded.imbalance_direction), + decoded.current_reference_price + ); + + const std::vector stream { + itch::Message {first_noii}, + itch::Message {itch::NOIIMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 1'000'000'000ULL, + .paired_shares = 60'000, + .imbalance_shares = 2'000, + .imbalance_direction = 'B', + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .far_price = 1'500'000, + .near_price = 1'500'000, + .current_reference_price = 1'500'000, + .cross_type = 'O', + .price_variation_indicator = 'L' + }}, + itch::Message {itch::CrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 2'000'000'000ULL, + .shares = 62'000, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .cross_price = 1'500'000, + .match_number = 1, + .cross_type = 'O' + }}, + }; + + std::vector buffer; + for (const auto& msg : stream) { + const auto frame = itch::encode_frame(msg); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + + itch::analytics::AuctionTracker tracker; + tracker.set_auction_callback([](const itch::analytics::Auction& auction) { + std::cout << std::format( + "{} for {}: {} shares @ {:.4f} (paired {}, imbalance {})\n", + itch::analytics::cross_type_name(auction.cross_type), + auction.stock, + auction.cross_shares, + auction.cross_price, + auction.paired_shares, + auction.imbalance_shares + ); + }); + + itch::Parser parser; + parser.parse(std::span {buffer}, [&](const itch::Message& msg) { + tracker.process(msg); + }); + + return 0; +} diff --git a/examples/analytics/bars_example.cpp b/examples/analytics/bars_example.cpp new file mode 100644 index 0000000..7b0cdd2 --- /dev/null +++ b/examples/analytics/bars_example.cpp @@ -0,0 +1,109 @@ +#include +#include +#include +#include +#include + +#include "itch/analytics/bars.hpp" +#include "itch/book/book_manager.hpp" +#include "itch/encoder.hpp" +#include "itch/parser.hpp" + +// Aggregates a synthetic trade tape into one-minute OHLCV bars with +// `BarBuilder` under a `TimeClock` policy, showing how the builder emits a +// completed bar each time the clock's bucket id advances. +auto main() -> int { + constexpr std::uint64_t base_timestamp {34'200'000'000'000ULL}; // 09:30:00.000 + constexpr std::uint64_t one_minute_ns {60'000'000'000ULL}; + + const std::vector trades { + itch::NonCrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp, + .order_reference_number = 1, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'500'000, + .match_number = 1 + }, + itch::NonCrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 10'000'000'000ULL, + .order_reference_number = 2, + .buy_sell_indicator = 'S', + .shares = 200, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'501'000, + .match_number = 2 + }, + itch::NonCrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + one_minute_ns + 10'000'000'000ULL, + .order_reference_number = 3, + .buy_sell_indicator = 'B', + .shares = 150, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'499'000, + .match_number = 3 + }, + itch::NonCrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + one_minute_ns + 30'000'000'000ULL, + .order_reference_number = 4, + .buy_sell_indicator = 'S', + .shares = 300, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'502'000, + .match_number = 4 + }, + itch::NonCrossTradeMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + (2 * one_minute_ns) + 10'000'000'000ULL, + .order_reference_number = 5, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'503'000, + .match_number = 5 + }, + }; + + std::vector buffer; + for (const auto& trade : trades) { + const auto frame = itch::encode_frame(trade); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + + itch::analytics::BarBuilder bars { + itch::analytics::TimeClock {.interval_ns = one_minute_ns}, + [](const itch::analytics::Bar& bar) { + std::cout << std::format( + "Bar {}: open={:.4f} high={:.4f} low={:.4f} close={:.4f} volume={} trades={}\n", + bar.bucket, + bar.open, + bar.high, + bar.low, + bar.close, + bar.volume, + bar.trade_count + ); + } + }; + + itch::book::BookManager manager; + manager.set_trade_callback([&](const itch::Trade& trade) { bars.add(trade); }); + + itch::Parser parser; + parser.parse(std::span {buffer}, [&](const itch::Message& msg) { + manager.process(msg); + }); + bars.flush(); // Emit the final, partial bar. + + return 0; +} diff --git a/examples/analytics/microstructure_example.cpp b/examples/analytics/microstructure_example.cpp new file mode 100644 index 0000000..372bd12 --- /dev/null +++ b/examples/analytics/microstructure_example.cpp @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include + +#include "itch/analytics/microstructure.hpp" +#include "itch/book/book_manager.hpp" +#include "itch/encoder.hpp" +#include "itch/parser.hpp" + +// Computes spread, mid price, queue imbalance, depth, and order-flow +// imbalance from a live `L3Book`, built from a small in-code sequence of +// resting orders so the example runs standalone without an external ITCH file. +auto main() -> int { + constexpr std::uint64_t base_timestamp {34'200'000'000'000ULL}; // 09:30:00.000 + + const std::vector orders { + itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp, + .order_reference_number = 1, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'499'000 + }, + itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 100, + .order_reference_number = 2, + .buy_sell_indicator = 'B', + .shares = 200, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'498'000 + }, + itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 200, + .order_reference_number = 3, + .buy_sell_indicator = 'S', + .shares = 150, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'501'000 + }, + itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 300, + .order_reference_number = 4, + .buy_sell_indicator = 'S', + .shares = 250, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'502'000 + }, + itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = base_timestamp + 400, + .order_reference_number = 5, + .buy_sell_indicator = 'B', + .shares = 300, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1'500'000 // Improves the best bid. + }, + }; + + std::vector buffer; + for (const auto& order : orders) { + const auto frame = itch::encode_frame(order); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + + itch::book::BookManager manager; + manager.track_symbol("AAPL"); + + std::vector snapshots; + manager.set_bbo_callback([&](const itch::book::L3Book&, const itch::book::Bbo& bbo) { + snapshots.push_back(bbo); + }); + + itch::Parser parser; + parser.parse(std::span {buffer}, [&](const itch::Message& msg) { + manager.process(msg); + }); + + const itch::book::L3Book* book = manager.book_for_symbol("AAPL"); + if (book == nullptr) { + std::cerr << "Error: book for AAPL was not created.\n"; + return 1; + } + + const itch::book::Bbo current = book->bbo(); + std::cout << std::format( + "Spread: {:.4f} Mid: {:.4f} Queue imbalance: {:.4f}\n", + itch::analytics::spread(current), + itch::analytics::mid_price(current), + itch::analytics::queue_imbalance(current) + ); + + const auto bid_depth = itch::analytics::depth_at_level(*book, itch::book::Side::buy, 2); + const auto ask_depth = itch::analytics::depth_at_level(*book, itch::book::Side::sell, 2); + std::cout << std::format("Depth (2 levels): bid={} ask={}\n", bid_depth, ask_depth); + + if (snapshots.size() >= 2) { + const double flow_imbalance = itch::analytics::order_flow_imbalance( + snapshots[snapshots.size() - 2], snapshots.back() + ); + std::cout << std::format( + "Order-flow imbalance (last BBO change): {:.2f}\n", flow_imbalance + ); + } + + return 0; +} diff --git a/examples/analytics/vwap_example.cpp b/examples/analytics/vwap_example.cpp new file mode 100644 index 0000000..0c7f4fd --- /dev/null +++ b/examples/analytics/vwap_example.cpp @@ -0,0 +1,50 @@ +#include +#include +#include +#include + +#include "itch/analytics/vwap.hpp" +#include "itch/book/book_manager.hpp" +#include "itch/parser.hpp" + +// Computes the session VWAP and TWAP for a chosen symbol by driving the +// BookManager's trade tape through running Vwap and Twap accumulators. +auto main(int argc, char* argv[]) -> int { + if (argc < 3) { + std::cerr << std::format("Usage: {} \n", argv[0]); + return 1; + } + + std::ifstream file {argv[1], std::ios::binary}; + if (!file) { + std::cerr << std::format("Error: cannot open '{}'.\n", argv[1]); + return 1; + } + std::vector buffer {std::istreambuf_iterator {file}, {}}; + + itch::book::BookManager manager; + manager.track_symbol(argv[2]); + + itch::analytics::Vwap vwap; + itch::analytics::Twap twap; + manager.set_trade_callback([&](const itch::Trade& trade) { + if (trade.symbol == argv[2]) { + vwap.add(trade.price, trade.shares); + twap.add(trade.price, trade.timestamp); + } + }); + + itch::Parser parser; + parser.parse(buffer.data(), buffer.size(), [&](const itch::Message& msg) { + manager.process(msg); + }); + + std::cout << std::format( + "VWAP for {}: {:.4f} over {} shares\n", argv[2], vwap.value(), vwap.volume() + ); + + // Twap integrates the prevailing trade price over elapsed time rather than + // volume, so it is reported separately from the volume-weighted figure above. + std::cout << std::format("TWAP for {}: {:.4f}\n", argv[2], twap.value()); + return 0; +} diff --git a/examples/book/book_engine_example.cpp b/examples/book/book_engine_example.cpp new file mode 100644 index 0000000..d9c5c19 --- /dev/null +++ b/examples/book/book_engine_example.cpp @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include + +#include "itch/book/book_manager.hpp" +#include "itch/parser.hpp" + +// Reconstructs every symbol's order book in one pass over an ITCH file using the +// multi-symbol BookManager, counting best-bid/offer changes and trades. +auto main(int argc, char* argv[]) -> int { + if (argc < 2) { + std::cerr << std::format("Usage: {} [symbol]\n", argv[0]); + return 1; + } + + std::ifstream file {argv[1], std::ios::binary}; + if (!file) { + std::cerr << std::format("Error: cannot open '{}'.\n", argv[1]); + return 1; + } + std::vector buffer {std::istreambuf_iterator {file}, {}}; + + itch::book::BookManager manager; + if (argc >= 3) { + manager.track_symbol(argv[2]); // Restrict to one symbol when requested. + } + + std::uint64_t bbo_updates = 0; + std::uint64_t trades = 0; + manager.set_bbo_callback([&](const itch::book::L3Book&, const itch::book::Bbo&) { + ++bbo_updates; + }); + manager.set_trade_callback([&](const itch::Trade&) { ++trades; }); + + itch::Parser parser; + parser.parse(buffer.data(), buffer.size(), [&](const itch::Message& msg) { + manager.process(msg); + }); + + std::cout << std::format( + "Reconstructed {} books, {} BBO changes, {} trades.\n", + manager.book_count(), + bbo_updates, + trades + ); + return 0; +} diff --git a/examples/encoder/encoder_example.cpp b/examples/encoder/encoder_example.cpp new file mode 100644 index 0000000..ddadf87 --- /dev/null +++ b/examples/encoder/encoder_example.cpp @@ -0,0 +1,77 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/parser.hpp" + +namespace { + +// ITCH text fields are fixed-width and space-padded on the wire, not +// null-terminated C strings, so a plain string-literal designated initializer +// cannot fill them (there is no room for the implicit '\0'). This copies +// `value` in and pads the remainder with spaces instead. +template +auto fill_field(char (&dest)[Size], std::string_view value) -> void { + std::ranges::fill(dest, ' '); + std::ranges::copy(value, dest); +} + +auto make_sample_order() -> itch::AddOrderMessage { + itch::AddOrderMessage order { + .stock_locate = 7, + .tracking_number = 3, + .timestamp = 34'200'000'123'456ULL, + .order_reference_number = 987654321, + .buy_sell_indicator = 'B', + .shares = 500, + .stock = {}, + .price = 1500000, + }; + fill_field(order.stock, "AAPL"); + return order; +} + +} // namespace + +// Demonstrates the encoder as the inverse of the parser: it serializes a message +// back to wire bytes and guarantees parse(encode(msg)) == msg, which is what +// lets tests and scenario generators synthesize valid ITCH streams instead of +// depending on captured fixture files. +auto main() -> int { + const itch::AddOrderMessage original = make_sample_order(); + const itch::Message message = original; + + const std::vector body = itch::encode_message(message); + const std::vector frame = itch::encode_frame(message); + std::cout << std::format( + "Encoded body: {} bytes; framed (2-byte length prefix): {} bytes\n", + body.size(), + frame.size() + ); + + itch::Parser parser; + const std::vector decoded_messages = + parser.parse(std::span {frame}); + std::cout << std::format( + "Parser decoded {} message(s) from the frame\n", decoded_messages.size() + ); + + // Message has no operator== of its own, so the round-trip guarantee is + // verified field by field on the decoded alternative instead. + const auto& decoded = std::get(decoded_messages.front()); + const bool round_trips = decoded.order_reference_number == original.order_reference_number && + decoded.price == original.price && decoded.shares == original.shares && + decoded.buy_sell_indicator == original.buy_sell_indicator && + decoded.timestamp == original.timestamp; + std::cout << std::format( + "Round-trip parse(encode(msg)) == msg: {}\n", round_trips ? "holds" : "FAILED" + ); + return 0; +} diff --git a/examples/io/arrow_export_example.cpp b/examples/io/arrow_export_example.cpp new file mode 100644 index 0000000..0f06c85 --- /dev/null +++ b/examples/io/arrow_export_example.cpp @@ -0,0 +1,88 @@ +#include +#include + +#ifdef ITCH_WITH_ARROW +#include +#include +#include +#include +#include + +#include "itch/io/arrow_export.hpp" +#include "itch/messages.hpp" + +namespace { + +// ITCH text fields are fixed-width and space-padded on the wire, not +// null-terminated C strings, so a plain string-literal designated initializer +// cannot fill them (there is no room for the implicit '\0'). This copies +// `value` in and pads the remainder with spaces instead. +template +auto fill_field(char (&dest)[Size], std::string_view value) -> void { + std::ranges::fill(dest, ' '); + std::ranges::copy(value, dest); +} + +} // namespace +#endif // ITCH_WITH_ARROW + +// Demonstrates ArrowExporter turning a handful of synthesized messages into a +// Parquet file, the columnar hand-off point for pandas/Polars/DuckDB/Spark +// pipelines. Only meaningful when the library is built with +// -DITCH_WITH_ARROW=ON; otherwise this file still compiles but does nothing. +auto main() -> int { +#ifdef ITCH_WITH_ARROW + const itch::SystemEventMessage system_event { + .stock_locate = 0, + .tracking_number = 1, + .timestamp = 34'200'000'000'000ULL, + .event_code = 'O', + }; + + itch::AddOrderMessage buy_order { + .stock_locate = 1, + .tracking_number = 1, + .timestamp = 34'200'000'500'000ULL, + .order_reference_number = 1001, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {}, + .price = 1500000, + }; + fill_field(buy_order.stock, "AAPL"); + + itch::AddOrderMessage sell_order { + .stock_locate = 1, + .tracking_number = 1, + .timestamp = 34'200'001'000'000ULL, + .order_reference_number = 1002, + .buy_sell_indicator = 'S', + .shares = 200, + .stock = {}, + .price = 1500500, + }; + fill_field(sell_order.stock, "AAPL"); + + const std::vector messages {system_event, buy_order, sell_order}; + + itch::io::ArrowExporter exporter; + for (const auto& message : messages) { + exporter.append(message); + } + + const auto parquet_path = + std::filesystem::temp_directory_path() / "itchcpp_arrow_export_example.parquet"; + const bool wrote_ok = exporter.write_parquet(parquet_path.string()); + if (!wrote_ok) { + std::cerr << std::format("Failed to write Parquet file: {}\n", exporter.error()); + return 1; + } + + std::cout << std::format( + "Appended {} rows and wrote Parquet file to {}\n", exporter.rows(), parquet_path.string() + ); +#else + std::cout << "This example requires -DITCH_WITH_ARROW=ON\n"; +#endif // ITCH_WITH_ARROW + return 0; +} diff --git a/examples/io/csv_sink_example.cpp b/examples/io/csv_sink_example.cpp new file mode 100644 index 0000000..df54c5f --- /dev/null +++ b/examples/io/csv_sink_example.cpp @@ -0,0 +1,89 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/io/csv_sink.hpp" +#include "itch/messages.hpp" + +namespace { + +// ITCH text fields are fixed-width and space-padded on the wire, not +// null-terminated C strings, so a plain string-literal designated initializer +// cannot fill them (there is no room for the implicit '\0'). This copies +// `value` in and pads the remainder with spaces instead. +template +auto fill_field(char (&dest)[Size], std::string_view value) -> void { + std::ranges::fill(dest, ' '); + std::ranges::copy(value, dest); +} + +auto make_sample_messages() -> std::vector { + const itch::SystemEventMessage system_event { + .stock_locate = 0, + .tracking_number = 1, + .timestamp = 34'200'000'000'000ULL, + .event_code = 'O', + }; + + itch::AddOrderMessage add_order { + .stock_locate = 1, + .tracking_number = 1, + .timestamp = 34'200'000'500'000ULL, + .order_reference_number = 1001, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {}, + .price = 1500000, + }; + fill_field(add_order.stock, "AAPL"); + + itch::StockTradingActionMessage trading_action { + .stock_locate = 1, + .tracking_number = 1, + .timestamp = 34'200'001'000'000ULL, + .stock = {}, + .trading_state = 'T', + .reserved = ' ', + .reason = {}, + }; + fill_field(trading_action.stock, "AAPL"); + fill_field(trading_action.reason, "T"); + + return {system_event, add_order, trading_action}; +} + +} // namespace + +// Demonstrates CsvSink as a concrete itch::io::MessageSink: every one of the 23 +// heterogeneous ITCH message structs flattens into the same wide CSV row schema, +// so downstream tooling never has to special-case which type it received. +auto main() -> int { + const std::vector messages = make_sample_messages(); + + const auto csv_path = std::filesystem::temp_directory_path() / "itchcpp_csv_sink_example.csv"; + std::ofstream out_file {csv_path}; + + itch::io::CsvSink sink {out_file}; + itch::io::MessageSink& generic_sink = sink; // exercise through the abstract interface + for (const auto& message : messages) { + generic_sink.write(message); + } + generic_sink.flush(); + out_file.close(); + + std::cout << std::format("Wrote {} CSV rows to {}\n", sink.rows_written(), csv_path.string()); + + std::ifstream in_file {csv_path}; + std::string line; + while (std::getline(in_file, line)) { + std::cout << line << '\n'; + } + return 0; +} diff --git a/examples/overlay/overlay_example.cpp b/examples/overlay/overlay_example.cpp new file mode 100644 index 0000000..663fa8a --- /dev/null +++ b/examples/overlay/overlay_example.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/overlay.hpp" + +// Demonstrates itch::overlay::for_each_message: a zero-copy, lazy alternative +// to itch::Parser. The eager Parser decodes every field of every message up +// front; the overlay view only converts the specific fields the callback +// touches, which pays off when a caller only cares about a few fields. +auto main() -> int { + const itch::Message first_order = itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200000000000ULL, + .order_reference_number = 3001, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1500000, + }; + const itch::Message second_order = itch::AddOrderMessage { + .stock_locate = 2, + .tracking_number = 0, + .timestamp = 34200100000000ULL, + .order_reference_number = 3002, + .buy_sell_indicator = 'S', + .shares = 200, + .stock = {'M', 'S', 'F', 'T', ' ', ' ', ' ', ' '}, + .price = 3200000, + }; + const itch::Message third_order = itch::AddOrderMessage { + .stock_locate = 3, + .tracking_number = 0, + .timestamp = 34200200000000ULL, + .order_reference_number = 3003, + .buy_sell_indicator = 'B', + .shares = 50, + .stock = {'G', 'O', 'O', 'G', ' ', ' ', ' ', ' '}, + .price = 2750000, + }; + + std::vector buffer; + for (const auto& message : {first_order, second_order, third_order}) { + const auto frame = itch::encode_frame(message); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + + std::uint64_t viewed_count = 0; + const auto delivered = + itch::overlay::for_each_message(buffer, [&](const itch::overlay::MessageView& view) { + if (view.type() != 'A') { + return; + } + ++viewed_count; + // Only stock() and price() are decoded here; every other field of + // this Add Order frame (shares, side, order reference number, ...) + // is never converted from network byte order, unlike a Parser pass. + const itch::overlay::AddOrderView order {view.data(), view.size()}; + std::cout << std::format( + "order {}: {} @ {}\n", viewed_count, order.stock(), order.price() + ); + }); + + std::cout << std::format( + "{} of {} frames viewed as Add Order messages\n", viewed_count, delivered + ); + return 0; +} diff --git a/examples/replay/replay_example.cpp b/examples/replay/replay_example.cpp new file mode 100644 index 0000000..bbd347a --- /dev/null +++ b/examples/replay/replay_example.cpp @@ -0,0 +1,84 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/replay.hpp" + +namespace { + +// ITCH text fields are fixed-width and space-padded on the wire, not +// null-terminated C strings, so a plain string-literal designated initializer +// cannot fill them (there is no room for the implicit '\0'). This copies +// `value` in and pads the remainder with spaces instead. +template +auto fill_field(char (&dest)[Size], std::string_view value) -> void { + std::ranges::fill(dest, ' '); + std::ranges::copy(value, dest); +} + +auto make_order(std::uint32_t index, std::uint64_t session_start_ns, std::uint64_t message_gap_ns) + -> itch::AddOrderMessage { + itch::AddOrderMessage order { + .stock_locate = 1, + .tracking_number = 1, + .timestamp = session_start_ns + static_cast(index) * message_gap_ns, + .order_reference_number = 1000ULL + static_cast(index), + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {}, + .price = 1'500'000U + (index * 100U), + }; + fill_field(order.stock, "AAPL"); + return order; +} + +} // namespace + +// Demonstrates ReplayEngine pacing a synthesized feed by its own timestamps +// rather than delivering it as fast as the CPU can parse, which is what makes +// it useful for realistic backtesting and system simulation instead of a plain +// Parser::parse loop. +auto main() -> int { + constexpr std::uint64_t session_start_ns = 34'200'000'000'000ULL; // 09:30:00.000000000 + constexpr std::uint64_t message_gap_ns = 100'000'000ULL; // 100 ms apart in feed time + constexpr std::uint32_t order_count = 4; + + std::vector stream; + for (std::uint32_t index = 0; index < order_count; ++index) { + const itch::AddOrderMessage order = make_order(index, session_start_ns, message_gap_ns); + const std::vector frame = itch::encode_frame(itch::Message {order}); + stream.insert(stream.end(), frame.begin(), frame.end()); + } + + // At 200x speed, the 100 ms feed-time gaps between messages become ~0.5 ms + // wall-clock sleeps, so the example finishes almost instantly while still + // exercising real pacing rather than a no-op. + itch::ReplayEngine engine {200.0}; + std::uint64_t messages_seen = 0; + const auto start = std::chrono::steady_clock::now(); + const std::uint64_t replayed = + engine.replay(std::span {stream}, [&](const itch::Message& message) { + ++messages_seen; + const auto& order = std::get(message); + std::cout << std::format( + "Replayed order #{} at {} ns past midnight\n", messages_seen, order.timestamp + ); + }); + const auto elapsed = std::chrono::steady_clock::now() - start; + + std::cout << std::format( + "Replayed {} messages in {} ms wall-clock at {}x speed\n", + replayed, + std::chrono::duration_cast(elapsed).count(), + engine.speed() + ); + return 0; +} diff --git a/examples/transport/moldudp64_example.cpp b/examples/transport/moldudp64_example.cpp new file mode 100644 index 0000000..15d9bce --- /dev/null +++ b/examples/transport/moldudp64_example.cpp @@ -0,0 +1,138 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/transport/moldudp64.hpp" +#include "itch/transport/sequencing.hpp" + +namespace { + +constexpr int BYTE_BITS = 8; +constexpr unsigned BYTE_MASK = 0xFF; +constexpr std::size_t SESSION_SIZE = 10; + +// Appends `width` bytes of `value` to `buffer`, most significant byte first. +auto append_big_endian(std::vector& buffer, std::uint64_t value, std::size_t width) + -> void { + for (std::size_t index = 0; index < width; ++index) { + const std::size_t shift = (width - 1 - index) * static_cast(BYTE_BITS); + buffer.push_back(static_cast((value >> shift) & BYTE_MASK)); + } +} + +// Builds one MoldUDP64 datagram: the 20-byte header followed by one +// length-prefixed message block (itch::encode_frame) per message. +auto make_packet( + std::string_view session, + std::uint64_t sequence_number, + const std::vector& messages +) -> std::vector { + std::string padded_session {session}; + padded_session.resize(SESSION_SIZE, ' '); + + std::vector packet; + for (const char letter : padded_session) { + packet.push_back(static_cast(letter)); + } + append_big_endian(packet, sequence_number, sizeof(std::uint64_t)); + append_big_endian(packet, messages.size(), sizeof(std::uint16_t)); + for (const auto& message : messages) { + const auto frame = itch::encode_frame(message); + packet.insert(packet.end(), frame.begin(), frame.end()); + } + return packet; +} + +// A minimal RetransmitRequester that only prints what it would ask for; a real +// implementation would issue a MoldUDP64 rewind/retransmit request here and +// feed the recovered messages back through decode_packet(). +class PrintingRetransmitRequester : public itch::transport::RetransmitRequester { + public: + auto request_retransmit( + std::string_view session, std::uint64_t start_sequence, std::uint64_t count + ) -> void override { + std::cout << std::format( + "retransmit request: session '{}', start {}, count {}\n", session, start_sequence, count + ); + } +}; + +} // namespace + +// Demonstrates decoding MoldUDP64 datagrams directly (no pcap capture +// involved): a normal packet followed by one that skips two sequence numbers, +// showing the embedded SequenceTracker detect the gap and drive a +// RetransmitRequester recovery hook. +auto main() -> int { + std::uint64_t decoded_count = 0; + itch::transport::MoldUdp64Decoder decoder {[&](const itch::Message& message) { + ++decoded_count; + const char type = std::visit([](const auto& msg) { return msg.message_type; }, message); + std::cout << std::format("decoded message {}: type '{}'\n", decoded_count, type); + }}; + + const itch::Message first_order = itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200000000000ULL, + .order_reference_number = 2001, + .buy_sell_indicator = 'B', + .shares = 300, + .stock = {'G', 'O', 'O', 'G', ' ', ' ', ' ', ' '}, + .price = 2750000, + }; + const itch::Message second_order = itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200100000000ULL, + .order_reference_number = 2002, + .buy_sell_indicator = 'S', + .shares = 150, + .stock = {'G', 'O', 'O', 'G', ' ', ' ', ' ', ' '}, + .price = 2751000, + }; + const itch::Message third_order = itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200200000000ULL, + .order_reference_number = 2003, + .buy_sell_indicator = 'B', + .shares = 75, + .stock = {'G', 'O', 'O', 'G', ' ', ' ', ' ', ' '}, + .price = 2749000, + }; + + const auto first_packet = make_packet("DEMO", 1, {first_order, second_order}); + decoder.decode_packet(first_packet); + + // Install the recovery hooks before feeding the gapped packet: sequence 1 + // carried 2 messages, so 3 is expected next, but the next packet starts at + // 5, leaving sequences 3 and 4 missing. + decoder.tracker().set_gap_callback( + [](std::string_view session, std::uint64_t expected, std::uint64_t received) { + std::cerr << std::format( + "gap on session '{}': expected {} but got {}\n", session, expected, received + ); + } + ); + PrintingRetransmitRequester requester; + decoder.tracker().set_retransmit_requester(&requester); + + const auto second_packet = make_packet("DEMO", 5, {third_order}); + decoder.decode_packet(second_packet); + + std::cout << std::format( + "{} packets decoded, {} messages decoded, {} gaps detected\n", + decoder.packets_decoded(), + decoder.messages_decoded(), + decoder.tracker().gap_count() + ); + return 0; +} diff --git a/examples/transport/pcap_replay_example.cpp b/examples/transport/pcap_replay_example.cpp new file mode 100644 index 0000000..5407d88 --- /dev/null +++ b/examples/transport/pcap_replay_example.cpp @@ -0,0 +1,51 @@ +#include +#include +#include +#include + +#include "itch/messages.hpp" +#include "itch/transport/pcap.hpp" + +// Replays an ITCH feed straight from a captured .pcap/.pcapng file: the reader +// unwraps the Ethernet/IP/UDP and MoldUDP64 framing and yields decoded ITCH +// messages, with sequence gaps surfaced through the embedded tracker. +auto main(int argc, char* argv[]) -> int { + if (argc < 2) { + std::cerr << std::format("Usage: {} [udp_port]\n", argv[0]); + return 1; + } + + std::uint64_t message_count = 0; + itch::transport::PcapReader reader {[&](const itch::Message& message) { + ++message_count; + const char type = std::visit([](const auto& msg) { return msg.message_type; }, message); + if (message_count <= 10) { + std::cout << std::format("message {:>3}: type '{}'\n", message_count, type); + } + }}; + + reader.mold_decoder().tracker().set_gap_callback( + [](std::string_view session, std::uint64_t expected, std::uint64_t received) { + std::cerr << std::format( + "gap on session '{}': expected {} but got {}\n", session, expected, received + ); + } + ); + + if (argc >= 3) { + reader.set_udp_port_filter(static_cast(std::stoul(argv[2]))); + } + + if (!reader.read_file(argv[1])) { + std::cerr << std::format("Error: '{}' is not a readable pcap/pcapng capture.\n", argv[1]); + return 1; + } + + std::cout << std::format( + "Decoded {} ITCH messages from {} UDP datagrams ({} missing messages detected).\n", + reader.messages_decoded(), + reader.udp_datagrams(), + reader.mold_decoder().tracker().gap_count() + ); + return 0; +} diff --git a/examples/transport/soupbintcp_example.cpp b/examples/transport/soupbintcp_example.cpp new file mode 100644 index 0000000..c3dd006 --- /dev/null +++ b/examples/transport/soupbintcp_example.cpp @@ -0,0 +1,133 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/transport/soupbintcp.hpp" + +namespace { + +constexpr int BYTE_BITS = 8; +constexpr unsigned BYTE_MASK = 0xFF; +constexpr std::size_t SESSION_SIZE = 10; +constexpr std::size_t SEQUENCE_SIZE = 20; + +// Appends a complete SoupBinTCP packet (2-byte big-endian length, 1-byte type, +// payload) to the byte stream under construction. +auto append_packet( + std::vector& stream, + itch::transport::SoupBinPacketType type, + std::span payload +) -> void { + const auto length = static_cast(payload.size() + 1); + stream.push_back(static_cast((length >> BYTE_BITS) & BYTE_MASK)); + stream.push_back(static_cast(length & BYTE_MASK)); + stream.push_back(static_cast(type)); + stream.insert(stream.end(), payload.begin(), payload.end()); +} + +// Builds the Login Accepted payload: a space-padded session id followed by an +// ASCII, space-padded starting sequence number, per the SoupBinTCP spec. +auto make_login_accepted_payload(std::string_view session, std::uint64_t sequence) + -> std::vector { + std::string text {session}; + text.resize(SESSION_SIZE, ' '); + std::string sequence_text = std::format("{}", sequence); + sequence_text.resize(SEQUENCE_SIZE, ' '); + text += sequence_text; + + std::vector payload(text.size()); + for (std::size_t index = 0; index < text.size(); ++index) { + payload[index] = static_cast(text[index]); + } + return payload; +} + +} // namespace + +// Demonstrates decoding a SoupBinTCP byte stream directly: a Login Accepted +// control packet followed by sequenced ITCH data packets, with one packet +// deliberately split across two feed() calls to show the decoder reassembling +// a packet that TCP delivered across multiple reads. +auto main() -> int { + std::uint64_t decoded_count = 0; + itch::transport::SoupBinDecoder decoder {[&](const itch::Message& message) { + ++decoded_count; + const char type = std::visit([](const auto& msg) { return msg.message_type; }, message); + std::cout << std::format("decoded message {}: type '{}'\n", decoded_count, type); + }}; + decoder.set_event_callback([](itch::transport::SoupBinPacketType type, + std::span payload) { + std::cout << std::format( + "control packet: type '{}' ({} bytes)\n", static_cast(type), payload.size() + ); + }); + + const itch::Message add_order = itch::AddOrderMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200000000000ULL, + .order_reference_number = 1001, + .buy_sell_indicator = 'B', + .shares = 100, + .stock = {'A', 'A', 'P', 'L', ' ', ' ', ' ', ' '}, + .price = 1500000, + }; + const itch::Message executed = itch::OrderExecutedMessage { + .stock_locate = 1, + .tracking_number = 0, + .timestamp = 34200500000000ULL, + .order_reference_number = 1001, + .executed_shares = 100, + .match_number = 5001, + }; + const itch::Message second_order = itch::AddOrderMessage { + .stock_locate = 2, + .tracking_number = 0, + .timestamp = 34201000000000ULL, + .order_reference_number = 1002, + .buy_sell_indicator = 'S', + .shares = 200, + .stock = {'M', 'S', 'F', 'T', ' ', ' ', ' ', ' '}, + .price = 3200000, + }; + + std::vector stream; + append_packet( + stream, + itch::transport::SoupBinPacketType::login_accepted, + make_login_accepted_payload("DEMO", 1) + ); + append_packet( + stream, itch::transport::SoupBinPacketType::sequenced_data, itch::encode_message(add_order) + ); + append_packet( + stream, itch::transport::SoupBinPacketType::sequenced_data, itch::encode_message(executed) + ); + append_packet( + stream, + itch::transport::SoupBinPacketType::sequenced_data, + itch::encode_message(second_order) + ); + + // Feed the stream in two chunks, splitting mid-packet, to prove feed() + // reassembles a packet spanning multiple TCP reads rather than requiring + // one call per packet. + const std::size_t split = stream.size() / 2; + decoder.feed(std::span {stream}.subspan(0, split)); + decoder.feed(std::span {stream}.subspan(split)); + + std::cout << std::format( + "session '{}', next sequence {}, {} messages decoded\n", + decoder.current_session(), + decoder.next_sequence(), + decoder.messages_decoded() + ); + return 0; +} diff --git a/examples/utilities/price_time_indicators_example.cpp b/examples/utilities/price_time_indicators_example.cpp new file mode 100644 index 0000000..24aad9b --- /dev/null +++ b/examples/utilities/price_time_indicators_example.cpp @@ -0,0 +1,53 @@ +#include +#include +#include +#include + +#include "itch/indicators.hpp" +#include "itch/price.hpp" +#include "itch/time.hpp" + +// The smallest, most beginner-friendly example in this directory: shows the +// three tiny utility types that keep raw ITCH wire values (integer prices, +// nanosecond timestamps, single-letter codes) from turning into silent +// off-by-a-decimal or look-the-code-up-yourself bugs in application code. +auto main() -> int { + // A raw ITCH price is just an integer with an implied decimal point: the + // same integer means different things depending on the field's scale, so + // dividing by 10000 (or 100000000) by hand everywhere risks picking the + // wrong divisor. StandardPrice/MwcbPrice bake the scale into the type. + const itch::StandardPrice trade_price = itch::make_price(1500000); // 4 implied decimals + const itch::MwcbPrice level1_price = + itch::make_mwcb_price(410250000000ULL); // 8 implied decimals + std::cout << std::format( + "Trade price raw {} -> {}\n", trade_price.raw(), trade_price.to_string() + ); + std::cout << std::format( + "MWCB level 1 raw {} -> {}\n", level1_price.raw(), level1_price.to_string() + ); + + // ITCH timestamps are raw nanoseconds since midnight with no date attached. + // These helpers turn that offset into readable clock time, or a full + // calendar time point given a session date, without hand-rolled div/mod math. + constexpr std::uint64_t market_open_ns = 34'200'000'000'000ULL; // 09:30:00.000000000 + std::cout << std::format( + "Market open time of day: {}\n", itch::format_timestamp(market_open_ns) + ); + const std::chrono::year_month_day session_date { + std::chrono::year {2026}, std::chrono::month {7}, std::chrono::day {10} + }; + std::cout << std::format( + "Market open full timestamp: {}\n", itch::format_time_point(session_date, market_open_ns) + ); + + // Single-letter wire codes (event codes, trading states, ...) are meaningless + // without a lookup table; indicators.hpp supplies compile-time tables instead + // of scattering string literals through every call site that needs one. + std::cout << std::format( + "Event code 'O' means: {}\n", itch::indicators::SYSTEM_EVENT_CODES.at_or('O', "Unknown") + ); + std::cout << std::format( + "Trading state 'T' means: {}\n", itch::indicators::TRADING_STATES.at_or('T', "Unknown") + ); + return 0; +} diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt new file mode 100644 index 0000000..3ead72a --- /dev/null +++ b/fuzz/CMakeLists.txt @@ -0,0 +1,26 @@ +cmake_minimum_required(VERSION 3.25) + +# libFuzzer ships with Clang. On other compilers the fuzz target is simply not +# created, so enabling the option elsewhere is a harmless no-op. +if (NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") + message(STATUS "Fuzzers require Clang (libFuzzer); skipping fuzz targets.") + return() +endif() + +# Instrument the library itself so the fuzzer gets coverage feedback through it, +# while leaving the main (non-fuzz) builds of the library untouched. +target_compile_options(itch PRIVATE -fsanitize=fuzzer-no-link,address,undefined) + +add_executable(parser_fuzzer parser_fuzzer.cpp) +target_link_libraries(parser_fuzzer PRIVATE itch::itch) +target_compile_options(parser_fuzzer PRIVATE -g -fsanitize=fuzzer,address,undefined) +target_link_options(parser_fuzzer PRIVATE -fsanitize=fuzzer,address,undefined) + +message(STATUS "Added 'parser_fuzzer' target") + +add_executable(moldudp64_fuzzer moldudp64_fuzzer.cpp) +target_link_libraries(moldudp64_fuzzer PRIVATE itch::itch) +target_compile_options(moldudp64_fuzzer PRIVATE -g -fsanitize=fuzzer,address,undefined) +target_link_options(moldudp64_fuzzer PRIVATE -fsanitize=fuzzer,address,undefined) + +message(STATUS "Added 'moldudp64_fuzzer' target") diff --git a/fuzz/moldudp64_fuzzer.cpp b/fuzz/moldudp64_fuzzer.cpp new file mode 100644 index 0000000..642b608 --- /dev/null +++ b/fuzz/moldudp64_fuzzer.cpp @@ -0,0 +1,28 @@ +#include +#include +#include + +#include "itch/transport/moldudp64.hpp" +#include "itch/transport/pcap.hpp" +#include "itch/transport/soupbintcp.hpp" + +// libFuzzer entry point for the transport decoders, which sit in front of the +// parser on untrusted network input. Arbitrary bytes are pushed through the +// MoldUDP64, SoupBinTCP, and pcap paths; any crash, overread, or sanitizer +// report is a genuine bug. None of these paths throw on malformed input by +// design, so there is nothing to catch. +extern "C" auto LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size) -> int { + const auto* bytes = static_cast(static_cast(data)); + const std::span buffer {bytes, size}; + + itch::transport::MoldUdp64Decoder mold {[](const itch::Message&) noexcept {}}; + mold.decode_packet(buffer); + + itch::transport::SoupBinDecoder soup {[](const itch::Message&) noexcept {}}; + soup.feed(buffer); + + itch::transport::PcapReader pcap {[](const itch::Message&) noexcept {}}; + pcap.read(buffer); + + return 0; +} diff --git a/fuzz/parser_fuzzer.cpp b/fuzz/parser_fuzzer.cpp new file mode 100644 index 0000000..daf62a2 --- /dev/null +++ b/fuzz/parser_fuzzer.cpp @@ -0,0 +1,26 @@ +#include +#include +#include +#include + +#include "itch/parser.hpp" + +// libFuzzer entry point. Feeds arbitrary bytes straight through the framing and +// dispatch path. Truncation is reported by `parse` as an exception, which is an +// expected outcome on random input and is swallowed; any crash, overread, or +// sanitizer report on this path is a genuine bug. +extern "C" auto LLVMFuzzerTestOneInput(const std::uint8_t* data, std::size_t size) -> int { + itch::Parser parser; + + // std::byte and the incoming bytes may legitimately alias; route through + // void* to avoid a forbidden reinterpret_cast. + const auto* bytes = static_cast(static_cast(data)); + const std::span buffer {bytes, size}; + + try { + parser.parse(buffer, [](const itch::Message&) noexcept {}); + } catch (const std::exception&) { + // Truncated input is expected and benign for the fuzzer. + } + return 0; +} diff --git a/include/itch/analytics/auctions.hpp b/include/itch/analytics/auctions.hpp new file mode 100644 index 0000000..58349b2 --- /dev/null +++ b/include/itch/analytics/auctions.hpp @@ -0,0 +1,115 @@ +#pragma once + +/// @file +/// @brief Reconstruction of Nasdaq auction (cross) events from the ITCH feed. +/// +/// Combines the Net Order Imbalance Indicator (NOII) messages leading up to a +/// cross with the Cross Trade message that prints it, producing a single +/// `Auction` record per opening, closing, halt, or IPO cross. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/analytics/imbalance.hpp" +#include "itch/indicators.hpp" +#include "itch/messages.hpp" +#include "itch/price.hpp" + +namespace itch::analytics { + +/// @brief A reconstructed auction (cross) event. +/// +/// Nasdaq runs the opening, closing, halt, and IPO crosses; their price discovery +/// is disseminated through the NOII (`I`) messages leading up to the cross and the +/// Cross Trade (`Q`) message that prints it. This record pairs the executed cross +/// with the imbalance state that immediately preceded it. +struct Auction { + std::uint64_t timestamp {0}; + std::uint16_t stock_locate {0}; + std::string stock; + char cross_type {'\0'}; ///< 'O' open, 'C' close, 'H' halt/pause, 'I' IPO. + StandardPrice cross_price {}; ///< The cross execution price. + std::uint64_t cross_shares {0}; ///< Shares executed in the cross. + std::uint64_t paired_shares {0}; ///< Paired shares from the latest NOII. + std::uint64_t imbalance_shares {0}; ///< Imbalance shares from the latest NOII. + char imbalance_direction {'\0'}; + bool had_imbalance {false}; ///< Whether NOII context was available. +}; + +/// @brief A human-readable description of a cross type code. +/// +/// @param cross_type The single-character cross type code (e.g. 'O', 'C', 'H', 'I'). +/// @return A short description of `cross_type`, or "Unknown" if unrecognized. +[[nodiscard]] inline auto cross_type_name(char cross_type) -> std::string_view { + constexpr indicators::FrozenMap types {std::to_array>({ + {'O', "Opening Cross"}, + {'C', "Closing Cross"}, + {'H', "Cross for halted/paused security"}, + {'I', "Intraday/IPO Cross"}, + })}; + return types.at_or(cross_type, "Unknown"); +} + +/// @brief Reconstructs auctions from the NOII and Cross Trade message stream. +/// +/// Feed every message with `process`: NOII (`I`) messages update the latest +/// imbalance state per security, and a Cross Trade (`Q`) finalizes an `Auction` +/// using the executed cross price together with that stored imbalance context, +/// which is delivered to the auction callback. +class AuctionTracker { + public: + using AuctionCallback = std::function; + + /// @brief Installs the callback invoked for each reconstructed auction. + /// + /// @param callback Function invoked with each `Auction` reconstructed from the stream. + auto set_auction_callback(AuctionCallback callback) -> void { + m_callback = std::move(callback); + } + + /// @brief Processes one ITCH message, emitting an auction on each cross. + /// + /// @param message The ITCH message to process; only NOII and Cross Trade messages + /// affect state. + auto process(const Message& message) -> void { + if (const auto* noii = std::get_if(&message)) { + m_latest_imbalance[noii->stock_locate] = make_imbalance_info(*noii); + return; + } + if (const auto* cross = std::get_if(&message)) { + Auction auction {}; + auction.timestamp = cross->timestamp; + auction.stock_locate = cross->stock_locate; + auction.stock = to_string(cross->stock, STOCK_LEN); + auction.cross_type = cross->cross_type; + auction.cross_price = StandardPrice {cross->cross_price}; + auction.cross_shares = cross->shares; + + const auto iter = m_latest_imbalance.find(cross->stock_locate); + if (iter != m_latest_imbalance.end()) { + auction.paired_shares = iter->second.paired_shares; + auction.imbalance_shares = iter->second.imbalance_shares; + auction.imbalance_direction = iter->second.imbalance_direction; + auction.had_imbalance = true; + } + if (m_callback) { + m_callback(auction); + } + return; + } + } + + private: + std::unordered_map m_latest_imbalance; + AuctionCallback m_callback {}; +}; + +} // namespace itch::analytics diff --git a/include/itch/analytics/bars.hpp b/include/itch/analytics/bars.hpp new file mode 100644 index 0000000..df9a012 --- /dev/null +++ b/include/itch/analytics/bars.hpp @@ -0,0 +1,149 @@ +#pragma once + +/// @file +/// @brief OHLCV bar aggregation from the trade tape under configurable clocks. +/// +/// `BarBuilder` consumes a stream of trades and emits completed bars whenever a +/// pluggable clock policy (time-, tick-, or volume-based) reports a new bucket, +/// letting time bars, tick bars, and volume bars share one implementation. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include + +#include "itch/price.hpp" +#include "itch/tape.hpp" + +namespace itch::analytics { + +/// @brief One OHLCV bar aggregated from the trade tape. +struct Bar { + std::uint64_t bucket {0}; ///< Clock bucket id this bar covers. + std::uint64_t start_timestamp {0}; ///< Timestamp of the first trade in the bar. + std::uint64_t end_timestamp {0}; ///< Timestamp of the last trade in the bar. + StandardPrice open {}; ///< First trade price. + StandardPrice high {}; ///< Highest trade price. + StandardPrice low {}; ///< Lowest trade price. + StandardPrice close {}; ///< Last trade price. + std::uint64_t volume {0}; ///< Total shares traded in the bar. + std::uint64_t trade_count {0}; ///< Number of trades in the bar. +}; + +/// @brief Callback invoked with each completed bar. +using BarCallback = std::function; + +/// @brief A clock that buckets trades by wall-clock time (nanoseconds). +struct TimeClock { + std::uint64_t interval_ns {1}; + + /// @brief Computes the bucket id for `trade` from its wall-clock timestamp. + /// + /// @param trade The trade whose timestamp determines the bucket. + /// @return The bucket id, `trade.timestamp / interval_ns`. + [[nodiscard]] auto bucket(const Trade& trade, std::uint64_t, std::uint64_t) const noexcept + -> std::uint64_t { + return trade.timestamp / interval_ns; + } +}; + +/// @brief A clock that buckets a fixed number of trades into each bar. +struct TickClock { + std::uint64_t ticks_per_bar {1}; + + /// @brief Computes the bucket id for the current trade from its tick position. + /// + /// @param tick_index 0-based index of the trade within the stream. + /// @return The bucket id, `tick_index / ticks_per_bar`. + [[nodiscard]] auto bucket(const Trade&, std::uint64_t, std::uint64_t tick_index) const noexcept + -> std::uint64_t { + return tick_index / ticks_per_bar; + } +}; + +/// @brief A clock that buckets a fixed amount of traded volume into each bar. +struct VolumeClock { + std::uint64_t volume_per_bar {1}; + + /// @brief Computes the bucket id for the current trade from cumulative volume. + /// + /// @param cumulative_volume Total shares traded so far, including this trade. + /// @return The bucket id, `cumulative_volume / volume_per_bar`. + [[nodiscard]] auto bucket(const Trade&, std::uint64_t cumulative_volume, std::uint64_t) + const noexcept -> std::uint64_t { + return cumulative_volume / volume_per_bar; + } +}; + +/// @brief Builds OHLCV bars from a trade stream under a configurable clock. +/// +/// Feed trades in timestamp order with `add`; each time the clock's bucket id +/// changes, the completed bar is emitted to the callback. `flush` emits the final, +/// partial bar. The clock is a small policy (see `TimeClock`, `TickClock`, +/// `VolumeClock`) so time-, tick-, and volume-based aggregation share one builder. +template +class BarBuilder { + public: + /// @brief Constructs a builder with the given clock policy and completion callback. + /// + /// @param clock Bucketing policy (e.g. `TimeClock`, `TickClock`, `VolumeClock`) + /// that decides when a bar completes. + /// @param callback Function invoked with each completed `Bar`. + BarBuilder(Clock clock, BarCallback callback) + : m_clock {std::move(clock)}, m_callback {std::move(callback)} {} + + /// @brief Adds one trade, emitting the previous bar when the bucket changes. + /// + /// @param trade The next trade in timestamp order to fold into the current bar. + auto add(const Trade& trade) -> void { + const std::uint64_t bucket = m_clock.bucket(trade, m_cumulative_volume, m_tick_index); + if (m_has_bar && bucket != m_bar.bucket) { + emit(); + } + if (!m_has_bar) { + m_bar = Bar {}; + m_bar.bucket = bucket; + m_bar.start_timestamp = trade.timestamp; + m_bar.open = trade.price; + m_bar.high = trade.price; + m_bar.low = trade.price; + m_has_bar = true; + } + m_bar.high = std::max(m_bar.high, trade.price); + m_bar.low = std::min(m_bar.low, trade.price); + m_bar.close = trade.price; + m_bar.end_timestamp = trade.timestamp; + m_bar.volume += trade.shares; + ++m_bar.trade_count; + + m_cumulative_volume += trade.shares; + ++m_tick_index; + } + + /// @brief Emits the current partial bar, if any, and clears it. + auto flush() -> void { + if (m_has_bar) { + emit(); + } + } + + private: + /// @brief Delivers the current bar to the callback and marks it inactive. + auto emit() -> void { + if (m_callback) { + m_callback(m_bar); + } + m_has_bar = false; + } + + Clock m_clock; + BarCallback m_callback; + Bar m_bar {}; + bool m_has_bar {false}; + std::uint64_t m_cumulative_volume {0}; + std::uint64_t m_tick_index {0}; +}; + +} // namespace itch::analytics diff --git a/include/itch/analytics/imbalance.hpp b/include/itch/analytics/imbalance.hpp new file mode 100644 index 0000000..707b75a --- /dev/null +++ b/include/itch/analytics/imbalance.hpp @@ -0,0 +1,78 @@ +#pragma once + +/// @file +/// @brief Decoded view of Net Order Imbalance Indicator (NOII) messages. +/// +/// Wraps the raw NOII (`I`) message fields in `ImbalanceInfo`, using strong price +/// types and a trimmed symbol, for use by auction reconstruction and other +/// imbalance-aware analytics. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include +#include + +#include "itch/indicators.hpp" +#include "itch/messages.hpp" +#include "itch/price.hpp" + +namespace itch::analytics { + +/// @brief A usable, decoded view of a Net Order Imbalance Indicator (`I`) message. +/// +/// The feed already carries the NOII data used during the opening and closing +/// crosses; this surfaces it with strong price types and a trimmed symbol instead +/// of the raw packed fields. +struct ImbalanceInfo { + std::uint64_t timestamp {0}; + std::uint16_t stock_locate {0}; + std::string stock; + std::uint64_t paired_shares {0}; + std::uint64_t imbalance_shares {0}; + char imbalance_direction {'\0'}; + StandardPrice far_price {}; + StandardPrice near_price {}; + StandardPrice current_reference_price {}; + char cross_type {'\0'}; + char price_variation_indicator {'\0'}; +}; + +/// @brief Builds an `ImbalanceInfo` from a raw NOII message. +/// +/// @param msg The raw NOII message to decode. +/// @return An `ImbalanceInfo` populated from `msg`'s fields. +[[nodiscard]] inline auto make_imbalance_info(const NOIIMessage& msg) -> ImbalanceInfo { + ImbalanceInfo info {}; + info.timestamp = msg.timestamp; + info.stock_locate = msg.stock_locate; + info.stock = to_string(msg.stock, STOCK_LEN); + info.paired_shares = msg.paired_shares; + info.imbalance_shares = msg.imbalance_shares; + info.imbalance_direction = msg.imbalance_direction; + info.far_price = StandardPrice {msg.far_price}; + info.near_price = StandardPrice {msg.near_price}; + info.current_reference_price = StandardPrice {msg.current_reference_price}; + info.cross_type = msg.cross_type; + info.price_variation_indicator = msg.price_variation_indicator; + return info; +} + +/// @brief A human-readable description of an imbalance direction code. +/// +/// @param direction The single-character imbalance direction code (e.g. 'B', 'S', 'N'). +/// @return A short description of `direction`, or "Unknown" if unrecognized. +[[nodiscard]] inline auto imbalance_direction_name(char direction) -> std::string_view { + constexpr indicators::FrozenMap directions {std::to_array>({ + {'B', "Buy imbalance"}, + {'S', "Sell imbalance"}, + {'N', "No imbalance"}, + {'O', "Insufficient orders to calculate"}, + {'P', "Paused"}, + })}; + return directions.at_or(direction, "Unknown"); +} + +} // namespace itch::analytics diff --git a/include/itch/analytics/microstructure.hpp b/include/itch/analytics/microstructure.hpp new file mode 100644 index 0000000..9c874e6 --- /dev/null +++ b/include/itch/analytics/microstructure.hpp @@ -0,0 +1,119 @@ +#pragma once + +/// @file +/// @brief Best-bid-offer microstructure metrics derived from the order book. +/// +/// Small, allocation-free functions that compute spread, mid price, queue +/// imbalance, depth, and order-flow imbalance from `Bbo`/`L3Book` snapshots. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include + +#include "itch/book/l3_book.hpp" + +namespace itch::analytics { + +/// @brief The bid-ask spread in price units, or NaN if either side is empty. +/// +/// @param bbo The best-bid-offer snapshot to compute the spread from. +/// @return `ask_price - bid_price`, or NaN if either side has no resting order. +[[nodiscard]] inline auto spread(const book::Bbo& bbo) -> double { + if (!bbo.has_bid || !bbo.has_ask) { + return std::numeric_limits::quiet_NaN(); + } + return bbo.ask_price.to_double() - bbo.bid_price.to_double(); +} + +/// @brief The mid price ((bid + ask) / 2), or NaN if either side is empty. +/// +/// @param bbo The best-bid-offer snapshot to compute the mid price from. +/// @return The average of the bid and ask prices, or NaN if either side has no +/// resting order. +[[nodiscard]] inline auto mid_price(const book::Bbo& bbo) -> double { + if (!bbo.has_bid || !bbo.has_ask) { + return std::numeric_limits::quiet_NaN(); + } + return (bbo.bid_price.to_double() + bbo.ask_price.to_double()) / 2.0; +} + +/// @brief Top-of-book queue imbalance in [-1, 1]. +/// +/// Positive values indicate more size resting on the bid than the offer +/// (buy-side pressure); negative values the reverse. Returns NaN when there is no +/// resting size at the top of either side. +/// +/// @param bbo The best-bid-offer snapshot to compute the queue imbalance from. +/// @return The normalized bid/ask size imbalance in [-1, 1], or NaN if both sides +/// are empty. +[[nodiscard]] inline auto queue_imbalance(const book::Bbo& bbo) -> double { + const double bid = static_cast(bbo.bid_shares); + const double ask = static_cast(bbo.ask_shares); + const double total = bid + ask; + if (total <= 0.0) { + return std::numeric_limits::quiet_NaN(); + } + return (bid - ask) / total; +} + +/// @brief Total displayed shares within the best `levels` price levels of a side. +/// +/// @param book The order book to query. +/// @param side The book side (bid or ask) to sum depth for. +/// @param levels Number of best price levels to include. +/// @return The total displayed shares across the requested levels. +[[nodiscard]] inline auto depth_at_level( + const book::L3Book& book, book::Side side, std::size_t levels +) -> std::uint64_t { + std::uint64_t total = 0; + for (const auto& level : book.depth(side, levels)) { + total += level.shares; + } + return total; +} + +/// @brief The order-flow imbalance between two consecutive BBO observations. +/// +/// Implements the standard order-flow-imbalance contribution of Cont, Kukanov and +/// Stoikov: each side compares the new best price and size against the previous to +/// attribute added or removed liquidity, and the offer contribution is subtracted +/// from the bid contribution. Positive values indicate net buy-side flow. +/// +/// @param previous The prior best-bid-offer snapshot. +/// @param current The current best-bid-offer snapshot. +/// @return The signed order-flow-imbalance contribution between the two snapshots. +[[nodiscard]] inline auto order_flow_imbalance(const book::Bbo& previous, const book::Bbo& current) + -> double { + const double prev_bid_price = previous.bid_price.to_double(); + const double curr_bid_price = current.bid_price.to_double(); + const double prev_ask_price = previous.ask_price.to_double(); + const double curr_ask_price = current.ask_price.to_double(); + const double prev_bid_size = static_cast(previous.bid_shares); + const double curr_bid_size = static_cast(current.bid_shares); + const double prev_ask_size = static_cast(previous.ask_shares); + const double curr_ask_size = static_cast(current.ask_shares); + + double bid_flow = 0.0; + if (curr_bid_price > prev_bid_price) { + bid_flow = curr_bid_size; + } else if (curr_bid_price == prev_bid_price) { + bid_flow = curr_bid_size - prev_bid_size; + } else { + bid_flow = -prev_bid_size; + } + + double ask_flow = 0.0; + if (curr_ask_price > prev_ask_price) { + ask_flow = prev_ask_size; + } else if (curr_ask_price == prev_ask_price) { + ask_flow = curr_ask_size - prev_ask_size; + } else { + ask_flow = -curr_ask_size; + } + + return bid_flow - ask_flow; +} + +} // namespace itch::analytics diff --git a/include/itch/analytics/vwap.hpp b/include/itch/analytics/vwap.hpp new file mode 100644 index 0000000..607cf97 --- /dev/null +++ b/include/itch/analytics/vwap.hpp @@ -0,0 +1,109 @@ +#pragma once + +/// @file +/// @brief Running VWAP and TWAP accumulators for streaming price and volume data. +/// +/// These lightweight accumulators are fed one execution or price sample at a time +/// and report a running volume-weighted or time-weighted average price without +/// retaining the full history of samples. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include + +#include "itch/price.hpp" + +namespace itch::analytics { + +/// @brief Running volume-weighted average price. +/// +/// Accumulates the sum of price times shares and the total shares; the value is +/// their ratio. Use `reset` at the start of each interval for an interval VWAP. +class Vwap { + public: + /// @brief Adds an execution of `shares` at `price` to the accumulation. + /// + /// @param price Execution price of the trade being added. + /// @param shares Number of shares traded at `price`. + auto add(StandardPrice price, std::uint64_t shares) -> void { + m_price_volume += price.to_double() * static_cast(shares); + m_volume += shares; + } + + /// @brief The current VWAP, or NaN when no volume has been added. + /// + /// @return The volume-weighted average price, or NaN if `volume()` is zero. + [[nodiscard]] auto value() const -> double { + if (m_volume == 0) { + return std::numeric_limits::quiet_NaN(); + } + return m_price_volume / static_cast(m_volume); + } + + /// @brief The total shares accumulated so far. + /// + /// @return The sum of shares passed to `add` since construction or the last `reset`. + [[nodiscard]] auto volume() const noexcept -> std::uint64_t { return m_volume; } + + /// @brief Clears the accumulation (start a new interval). + auto reset() noexcept -> void { + m_price_volume = 0.0; + m_volume = 0; + } + + private: + double m_price_volume {0.0}; + std::uint64_t m_volume {0}; +}; + +/// @brief Running time-weighted average price. +/// +/// Integrates the prevailing price over time: each sample contributes its price +/// weighted by the elapsed time since the previous sample. The value is the +/// time-weighted mean over the observed span. +class Twap { + public: + /// @brief Records that the price became `price` at `timestamp` (ns). + /// + /// @param price The prevailing price starting at `timestamp`. + /// @param timestamp Time of the price change, in nanoseconds. + auto add(StandardPrice price, std::uint64_t timestamp) -> void { + if (m_has_sample && timestamp > m_last_timestamp) { + const double elapsed = static_cast(timestamp - m_last_timestamp); + m_price_time += m_last_price * elapsed; + m_total_time += elapsed; + } + m_last_price = price.to_double(); + m_last_timestamp = timestamp; + m_has_sample = true; + } + + /// @brief The current TWAP, or NaN when no span has elapsed. + /// + /// @return The time-weighted average price, or NaN if no sample has been added. + [[nodiscard]] auto value() const -> double { + if (m_total_time <= 0.0) { + return m_has_sample ? m_last_price : std::numeric_limits::quiet_NaN(); + } + return m_price_time / m_total_time; + } + + /// @brief Clears the accumulation (start a new interval). + auto reset() noexcept -> void { + m_price_time = 0.0; + m_total_time = 0.0; + m_last_price = 0.0; + m_last_timestamp = 0; + m_has_sample = false; + } + + private: + double m_price_time {0.0}; + double m_total_time {0.0}; + double m_last_price {0.0}; + std::uint64_t m_last_timestamp {0}; + bool m_has_sample {false}; +}; + +} // namespace itch::analytics diff --git a/include/itch/book/book_manager.hpp b/include/itch/book/book_manager.hpp new file mode 100644 index 0000000..54766b5 --- /dev/null +++ b/include/itch/book/book_manager.hpp @@ -0,0 +1,170 @@ +#pragma once + +/// @file +/// @brief Full-market order book manager that fans out ITCH messages to +/// per-symbol `L3Book` instances. +/// +/// This header declares `BookManager`, which owns a locate-indexed table of +/// `L3Book`s, routes each parsed ITCH message to the right book, optionally +/// restricts work to a configured symbol universe, and emits best-bid/offer +/// and trade events as they occur. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include +#include +#include +#include + +#include "itch/book/l3_book.hpp" +#include "itch/messages.hpp" +#include "itch/tape.hpp" + +namespace itch::book { + +/// @brief Maintains a full-market set of order books from a single pass over the +/// feed. +/// +/// Every order, execution, cancel, delete, and replace message carries a stock +/// locate code; the manager routes each message to that security's `L3Book` in +/// O(1) through a flat, locate-indexed table, reconstructing every symbol on the +/// feed at once rather than a single pre-selected one. It optionally restricts +/// work to a chosen universe of symbols, emits best-bid/offer change events as +/// they happen, and extracts the trade tape. +class BookManager { + public: + /// @brief Invoked when a book's best bid or offer changes. + using BboCallback = std::function; + + /// @brief Constructs an empty manager with no books tracked yet. + BookManager() = default; + + /// @brief Processes one parsed ITCH message, updating the relevant book and + /// emitting BBO/trade events as appropriate. + /// @param message The parsed ITCH message to apply. + auto process(const Message& message) -> void; + + /// @brief Installs the best-bid/offer change callback (empty clears it). + /// @param callback Invoked whenever a tracked book's best bid or offer + /// changes. + auto set_bbo_callback(BboCallback callback) -> void { m_bbo_callback = std::move(callback); } + + /// @brief Installs the trade-tape callback (empty clears it). + /// @param callback Invoked for each trade extracted from the feed. + auto set_trade_callback(TradeCallback callback) -> void { + m_trade_callback = std::move(callback); + } + + /// @brief Restricts tracking to the given symbol (call once per symbol). When + /// no symbol is added, every symbol on the feed is tracked. + /// @param symbol The ticker symbol to add to the tracked universe. + auto track_symbol(std::string symbol) -> void { m_universe.insert(std::move(symbol)); } + + /// @brief The book for a locate code, or nullptr if none is tracked. + /// @param stock_locate The exchange-assigned stock locate code. + /// @return Pointer to the book for `stock_locate`, or nullptr if it is + /// not tracked. + [[nodiscard]] auto book(std::uint16_t stock_locate) const -> const L3Book*; + + /// @brief The book for a symbol, or nullptr if none is tracked. + /// @param symbol The ticker symbol to look up. + /// @return Pointer to the book for `symbol`, or nullptr if it is not + /// tracked. + [[nodiscard]] auto book_for_symbol(std::string_view symbol) const -> const L3Book*; + + /// @brief The number of books currently maintained. + /// @return The count of books currently tracked. + [[nodiscard]] auto book_count() const noexcept -> std::size_t { return m_book_count; } + + /// @brief The symbol associated with a locate code (empty if unknown). + /// @param stock_locate The exchange-assigned stock locate code. + /// @return The ticker symbol for `stock_locate`, or an empty view if + /// unknown. + [[nodiscard]] auto symbol_for_locate(std::uint16_t stock_locate) const -> std::string_view; + + private: + struct BookEntry { + L3Book book; + Bbo last_bbo {}; + }; + + /// @brief Returns the entry for a locate, creating it if the symbol is + /// in-universe. + /// @param stock_locate The exchange-assigned stock locate code. + /// @param symbol The ticker symbol associated with `stock_locate`. + /// @return Pointer to the (possibly newly created) entry, or nullptr if + /// `symbol` is not in the tracked universe. + auto ensure_entry(std::uint16_t stock_locate, std::string_view symbol) -> BookEntry*; + + /// @brief Returns the existing entry for a locate, or nullptr. + /// @param stock_locate The exchange-assigned stock locate code. + /// @return Pointer to the entry for `stock_locate`, or nullptr if none + /// exists. + [[nodiscard]] auto entry(std::uint16_t stock_locate) const -> BookEntry*; + + /// @brief Emits a BBO event if the book's top has changed since last seen. + /// @param target The book entry to check and, if changed, report. + auto emit_bbo_if_changed(BookEntry& target) -> void; + + /// @brief Whether the symbol should be tracked given the configured + /// universe. + /// @param symbol The ticker symbol to check. + /// @return True if `symbol` should be tracked, false otherwise. + [[nodiscard]] auto in_universe(std::string_view symbol) const -> bool; + + /// @brief Adds a new order to the appropriate book from an Add Order (or + /// MPID-attributed Add Order) message. + /// @tparam AddMessage AddOrderMessage or AddOrderMPIDAttributionMessage. + /// @param add The parsed add-order message. + template + auto handle_add_order(const AddMessage& add) -> void; + + /// @brief Executes shares against a resting order and emits a trade using + /// the order's own price (Order Executed carries no price itself). + /// @param exec The parsed order-executed message. + auto handle_order_executed(const OrderExecutedMessage& exec) -> void; + + /// @brief Executes shares against a resting order at an explicit price + /// and emits a trade (Order Executed With Price). + /// @param exec The parsed order-executed-with-price message. + auto handle_order_executed_with_price(const OrderExecutedWithPriceMessage& exec) -> void; + + /// @brief Reduces a resting order's size (Order Cancel). + /// @param cancel The parsed order-cancel message. + auto handle_order_cancel(const OrderCancelMessage& cancel) -> void; + + /// @brief Removes a resting order from its book (Order Delete). + /// @param del The parsed order-delete message. + auto handle_order_delete(const OrderDeleteMessage& del) -> void; + + /// @brief Replaces a resting order with a new reference number, size, and + /// price (Order Replace). + /// @param replace The parsed order-replace message. + auto handle_order_replace(const OrderReplaceMessage& replace) -> void; + + /// @brief Extracts a trade-tape event from a non-displayed trade print + /// (Non-Cross Trade); does not alter the visible book. + /// @param trade The parsed non-cross-trade message. + auto handle_non_cross_trade(const NonCrossTradeMessage& trade) -> void; + + /// @brief Extracts a trade-tape event from a cross trade (Cross Trade). + /// @param cross The parsed cross-trade message. + auto handle_cross_trade(const CrossTradeMessage& cross) -> void; + + /// @brief Records the symbol associated with a locate code (Stock + /// Directory). + /// @param directory The parsed stock-directory message. + auto handle_stock_directory(const StockDirectoryMessage& directory) -> void; + + std::vector> m_books_by_locate; ///< Indexed by locate. + std::vector m_symbol_by_locate; ///< Locate -> symbol. + std::unordered_set m_universe; ///< Empty == track all. + BboCallback m_bbo_callback {}; + TradeCallback m_trade_callback {}; + std::size_t m_book_count {0}; +}; + +} // namespace itch::book diff --git a/include/itch/book/l3_book.hpp b/include/itch/book/l3_book.hpp new file mode 100644 index 0000000..ccf3c90 --- /dev/null +++ b/include/itch/book/l3_book.hpp @@ -0,0 +1,251 @@ +#pragma once + +/// @file +/// @brief Single-symbol, order-level (L3) limit order book with +/// allocation-light internals. +/// +/// This header declares `L3Book`, which reconstructs one security's full +/// order book from ITCH add/execute/cancel/delete/replace messages using an +/// object pool and intrusive FIFO queues instead of per-order heap +/// allocation, plus the small `Side`, `DepthLevel`, `OrderView`, and `Bbo` +/// value types used to query it. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include + +#include "itch/book/order_index.hpp" +#include "itch/price.hpp" + +namespace itch::book { + +/// @brief Which side of the book an order rests on. +enum class Side : char { + buy = 'B', ///< A bid. + sell = 'S', ///< An offer/ask. +}; + +/// @brief Aggregated state of one price level, for L2 depth snapshots. +struct DepthLevel { + StandardPrice price {}; ///< The level's limit price. + std::uint64_t shares {0}; ///< Total displayed shares resting at the level. + std::uint32_t order_count {0}; ///< Number of resting orders at the level. +}; + +/// @brief A single resting order, for L3 order-level snapshots. +struct OrderView { + std::uint64_t reference_number {0}; ///< Exchange order reference number. + std::uint32_t shares {0}; ///< Shares still resting. + StandardPrice price {}; ///< Limit price. +}; + +/// @brief Best bid and offer of a single book. +struct Bbo { + bool has_bid {false}; ///< Whether a bid side exists. + bool has_ask {false}; ///< Whether an ask side exists. + StandardPrice bid_price {}; ///< Best (highest) bid price. + std::uint64_t bid_shares {0}; ///< Shares at the best bid. + StandardPrice ask_price {}; ///< Best (lowest) ask price. + std::uint64_t ask_shares {0}; ///< Shares at the best ask. + + /// @brief Compares two BBO snapshots for equality of all fields. + /// @param lhs The first snapshot to compare. + /// @param rhs The second snapshot to compare. + /// @return True if `lhs` and `rhs` have equal has_bid/has_ask/prices/ + /// shares, false otherwise. + [[nodiscard]] friend auto operator==(const Bbo&, const Bbo&) noexcept -> bool = default; +}; + +/// @brief A single-symbol, order-level (L3) limit order book with allocation-light +/// internals. +/// +/// Unlike the original `LimitOrderBook`, which stored each order in a +/// `std::shared_ptr` inside a `std::list` and each price level in a `std::map`, +/// this engine holds orders in a reusable object pool (a flat vector with a free +/// list) linked into intrusive FIFO queues, and keeps each side's price levels in +/// a flat, sorted vector. There is no per-order heap allocation, no atomic +/// refcount, and no per-level node allocation on the hot path; order lookup by +/// reference number is O(1) and the best bid/offer is the front of a ladder. +class L3Book { + public: + /// @brief Constructs a book, optionally tagged with its stock symbol. + /// @param symbol The ticker symbol to associate with the book (may be + /// empty). + explicit L3Book(std::string symbol = {}); + + /// @brief Sets the stock symbol associated with this book. + /// @param symbol The ticker symbol to associate with the book. + auto set_symbol(std::string symbol) -> void { m_symbol = std::move(symbol); } + + /// @brief The stock symbol associated with this book (may be empty). + /// @return Const reference to the associated ticker symbol. + [[nodiscard]] auto symbol() const noexcept -> const std::string& { return m_symbol; } + + /// @brief Adds a new resting order to the book (ITCH `A`/`F`). + /// @param reference_number Exchange order reference number identifying + /// the new order. + /// @param side The side of the book the order rests on. + /// @param shares The number of shares in the new order. + /// @param price The raw (unscaled) limit price of the new order. + auto add_order( + std::uint64_t reference_number, Side side, std::uint32_t shares, std::uint32_t price + ) -> void; + + /// @brief Removes `shares` from an order on execution (ITCH `E`/`C`), + /// deleting it when fully filled. Returns the shares actually removed. + /// @param reference_number Exchange order reference number of the order + /// being executed against. + /// @param shares The number of shares to remove from the order. + /// @return The number of shares actually removed (may be less than + /// `shares` if the order does not have that many resting). + auto execute_order(std::uint64_t reference_number, std::uint32_t shares) -> std::uint32_t; + + /// @brief Removes `shares` from an order on a partial cancel (ITCH `X`), + /// deleting it when it reaches zero. Returns the shares actually removed. + /// @param reference_number Exchange order reference number of the order + /// being reduced. + /// @param shares The number of shares to remove from the order. + /// @return The number of shares actually removed (may be less than + /// `shares` if the order does not have that many resting). + auto reduce_order(std::uint64_t reference_number, std::uint32_t shares) -> std::uint32_t; + + /// @brief Deletes an order in its entirety (ITCH `D`). + /// @param reference_number Exchange order reference number of the order + /// to delete. + auto delete_order(std::uint64_t reference_number) -> void; + + /// @brief Replaces an order with a new reference number, size, and price + /// on the same side (ITCH `U`). + /// @param old_reference_number Exchange order reference number of the + /// order being replaced. + /// @param new_reference_number Exchange order reference number assigned + /// to the replacement order. + /// @param shares The number of shares in the replacement order. + /// @param price The raw (unscaled) limit price of the replacement order. + auto replace_order( + std::uint64_t old_reference_number, + std::uint64_t new_reference_number, + std::uint32_t shares, + std::uint32_t price + ) -> void; + + /// @brief Whether an order with the given reference number is resting. + /// @param reference_number Exchange order reference number to look up. + /// @return True if an order with `reference_number` is resting, false + /// otherwise. + [[nodiscard]] auto contains(std::uint64_t reference_number) const -> bool; + + /// @brief The raw limit price of a resting order, if present. + /// @param reference_number Exchange order reference number to look up. + /// @return The order's raw limit price, or `std::nullopt` if no such + /// order is resting. + [[nodiscard]] auto order_price(std::uint64_t reference_number + ) const -> std::optional; + + /// @brief The side of a resting order, if present. + /// @param reference_number Exchange order reference number to look up. + /// @return The order's side, or `std::nullopt` if no such order is + /// resting. + [[nodiscard]] auto order_side(std::uint64_t reference_number) const -> std::optional; + + /// @brief The current best bid and offer. + /// @return The book's current best bid and offer snapshot. + [[nodiscard]] auto bbo() const -> Bbo; + + /// @brief Aggregated L2 depth for a side, best level first. + /// + /// @param side The side to report. + /// @param max_levels The maximum number of levels to return (0 == all). + /// @return The requested side's price levels, best (top of book) first. + [[nodiscard]] auto depth(Side side, std::size_t max_levels = 0) const + -> std::vector; + + /// @brief The resting orders at a given price on a side, in time priority. + /// @param side The side to query. + /// @param price The raw (unscaled) limit price to query. + /// @return The resting orders at `price` on `side`, oldest first. + [[nodiscard]] auto orders_at(Side side, std::uint32_t price) const -> std::vector; + + /// @brief The number of active price levels on a side. + /// @param side The side to query. + /// @return The count of active price levels on `side`. + [[nodiscard]] auto level_count(Side side) const noexcept -> std::size_t; + + /// @brief Whether the book has no resting orders on either side. + /// @return True if no orders are resting on either side, false otherwise. + [[nodiscard]] auto empty() const noexcept -> bool { return m_index.empty(); } + + private: + /// @brief Sentinel index meaning "no node". + static constexpr std::uint32_t NIL = 0xFFFFFFFFU; + + /// @brief One resting order in the object pool, linked into a level's FIFO. + struct OrderNode { + std::uint64_t reference_number {0}; + std::uint32_t shares {0}; + std::uint32_t price {0}; + Side side {Side::buy}; + std::uint32_t next {NIL}; ///< Next order in level FIFO, or next free node. + std::uint32_t prev {NIL}; ///< Previous order in level FIFO. + }; + + /// @brief One price level holding the head/tail of an intrusive FIFO queue. + struct Level { + std::uint32_t price {0}; + std::uint64_t total_shares {0}; + std::uint32_t order_count {0}; + std::uint32_t head {NIL}; + std::uint32_t tail {NIL}; + }; + + /// @brief The mutable level ladder for a side. + /// @param side The side whose ladder to return. + /// @return Reference to the vector of price levels for `side`. + [[nodiscard]] auto side_levels(Side side) noexcept -> std::vector&; + + /// @brief The level ladder for a side. + /// @param side The side whose ladder to return. + /// @return Const reference to the vector of price levels for `side`. + [[nodiscard]] auto side_levels(Side side) const noexcept -> const std::vector&; + + /// @brief Allocates a node from the free list (or grows the pool) and + /// returns its index. + /// @return The pool index of the newly allocated node. + auto allocate_node() -> std::uint32_t; + + /// @brief Returns a node to the free list. + /// @param node_index The pool index of the node to free. + auto free_node(std::uint32_t node_index) -> void; + + /// @brief Finds the index of the level at `price` on `side`, or NIL if + /// absent. + /// @param side The side to search. + /// @param price The raw (unscaled) limit price to find. + /// @return The index of the matching level, or `NIL` if none exists. + [[nodiscard]] auto find_level(Side side, std::uint32_t price) const -> std::uint32_t; + + /// @brief Finds, or inserts in sorted order, the level at `price`; + /// returns its index. + /// @param side The side to search or insert into. + /// @param price The raw (unscaled) limit price to find or create. + /// @return The index of the existing or newly created level. + auto find_or_create_level(Side side, std::uint32_t price) -> std::uint32_t; + + /// @brief Unlinks a node from its level FIFO and removes the level if it + /// empties. + /// @param node_index The pool index of the node to unlink. + auto unlink_node(std::uint32_t node_index) -> void; + + std::string m_symbol; + std::vector m_pool; + std::uint32_t m_free_head {NIL}; + std::vector m_bids; ///< Sorted high to low; front is the best bid. + std::vector m_asks; ///< Sorted low to high; front is the best ask. + // Reference-number -> pool index for O(1), allocation-free order lookup. + OrderIndex m_index; +}; + +} // namespace itch::book diff --git a/include/itch/book/order_index.hpp b/include/itch/book/order_index.hpp new file mode 100644 index 0000000..fc73e18 --- /dev/null +++ b/include/itch/book/order_index.hpp @@ -0,0 +1,176 @@ +#pragma once + +/// @file +/// @brief Allocation-light, open-addressed hash map from order reference +/// number to order-pool index. +/// +/// This header declares `OrderIndex`, the lookup structure `L3Book` uses to +/// resolve an ITCH order reference number to its slot in the order pool in +/// O(1) without the per-insert/per-erase heap allocations of +/// `std::unordered_map`. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include + +namespace itch::book { + +/// @brief A flat, open-addressed hash map from order reference number to pool +/// index, used for O(1) order lookup without per-order heap allocation. +/// +/// `std::unordered_map` allocates a node on every insert and frees one on every +/// erase, which on a feed of hundreds of millions of add/delete messages dominates +/// the book's cost and defeats the allocation-free goal. This map stores its slots +/// in a single contiguous vector, probes linearly (cache friendly), and uses +/// backward-shift deletion so heavy add/cancel churn does not accumulate +/// tombstones. It only allocates when it grows. +class OrderIndex { + public: + /// @brief Sentinel returned by `find` when a key is absent. + static constexpr std::uint32_t NPOS = 0xFFFFFFFFU; + + /// @brief Constructs an empty map with the initial table capacity + /// pre-allocated. + OrderIndex() { rehash(INITIAL_CAPACITY); } + + /// @brief The number of stored keys. + /// @return The count of stored keys. + [[nodiscard]] auto size() const noexcept -> std::size_t { return m_count; } + + /// @brief Whether the map holds no keys. + /// @return True if the map holds no keys, false otherwise. + [[nodiscard]] auto empty() const noexcept -> bool { return m_count == 0; } + + /// @brief Returns the value for `key`, or `NPOS` if absent. + /// @param key The order reference number to look up. + /// @return The stored pool index for `key`, or `NPOS` if absent. + [[nodiscard]] auto find(std::uint64_t key) const noexcept -> std::uint32_t { + std::size_t slot = hash(key) & m_mask; + while (m_slots[slot].used) { + if (m_slots[slot].key == key) { + return m_slots[slot].value; + } + slot = (slot + 1) & m_mask; + } + return NPOS; + } + + /// @brief Whether `key` is present. + /// @param key The order reference number to check. + /// @return True if `key` is present, false otherwise. + [[nodiscard]] auto contains(std::uint64_t key) const noexcept -> bool { + return find(key) != NPOS; + } + + /// @brief Inserts or overwrites the value for `key`. + /// @param key The order reference number to insert or update. + /// @param value The pool index to associate with `key`. + auto insert(std::uint64_t key, std::uint32_t value) -> void { + if ((m_count + 1) * LOAD_FACTOR_DEN >= m_slots.size() * LOAD_FACTOR_NUM) { + rehash(m_slots.size() * 2); + } + std::size_t slot = hash(key) & m_mask; + while (m_slots[slot].used) { + if (m_slots[slot].key == key) { + m_slots[slot].value = value; + return; + } + slot = (slot + 1) & m_mask; + } + m_slots[slot] = Slot {key, value, true}; + ++m_count; + } + + /// @brief Removes `key` if present, repairing the probe chain in place. + /// @param key The order reference number to remove. + auto erase(std::uint64_t key) -> void { + std::size_t slot = hash(key) & m_mask; + while (m_slots[slot].used) { + if (m_slots[slot].key == key) { + remove_at(slot); + return; + } + slot = (slot + 1) & m_mask; + } + } + + /// @brief Drops all keys (retaining capacity). + auto clear() -> void { + for (auto& slot : m_slots) { + slot.used = false; + } + m_count = 0; + } + + private: + struct Slot { + std::uint64_t key {0}; + std::uint32_t value {0}; + bool used {false}; + }; + + static constexpr std::size_t INITIAL_CAPACITY = 1024; + static constexpr std::size_t LOAD_FACTOR_NUM = 7; // Grow past 70% load. + static constexpr std::size_t LOAD_FACTOR_DEN = 10; + + /// @brief Computes a finalizing hash mix (splitmix64) of a key so dense, + /// sequential reference numbers spread across the table instead of + /// clustering. + /// @param key The order reference number to hash. + /// @return The mixed hash value used to select a probe slot. + [[nodiscard]] static auto hash(std::uint64_t key) noexcept -> std::size_t { + key ^= key >> 33; + key *= 0xFF51AFD7ED558CCDULL; + key ^= key >> 33; + key *= 0xC4CEB9FE1A85EC53ULL; + key ^= key >> 33; + return static_cast(key); + } + + /// @brief Removes the entry at `hole` and backward-shifts any entries in + /// its probe chain so lookups for other keys are not broken by the + /// gap. + /// @param hole The slot index of the entry to remove. + auto remove_at(std::size_t hole) -> void { + m_slots[hole].used = false; + --m_count; + std::size_t next = (hole + 1) & m_mask; + while (m_slots[next].used) { + const std::size_t home = hash(m_slots[next].key) & m_mask; + // Move the entry into the hole if its home position is not within the + // cyclic interval (hole, next], i.e. the hole sits between its home + // and its current slot. + const bool movable = + (hole <= next) ? (home <= hole || home > next) : (home <= hole && home > next); + if (movable) { + m_slots[hole] = m_slots[next]; + m_slots[next].used = false; + hole = next; + } + next = (next + 1) & m_mask; + } + } + + /// @brief Grows (or initializes) the table to `new_capacity` and + /// re-inserts every previously stored key. + /// @param new_capacity The new table capacity, must be a power of two. + auto rehash(std::size_t new_capacity) -> void { + std::vector old = std::move(m_slots); + m_slots.assign(new_capacity, Slot {}); + m_mask = new_capacity - 1; + m_count = 0; + for (const auto& slot : old) { + if (slot.used) { + insert(slot.key, slot.value); + } + } + } + + std::vector m_slots; + std::size_t m_count {0}; + std::size_t m_mask {0}; +}; + +} // namespace itch::book diff --git a/include/itch/detail/wire.hpp b/include/itch/detail/wire.hpp new file mode 100644 index 0000000..33cc4c6 --- /dev/null +++ b/include/itch/detail/wire.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include +#include + +#include "itch/messages.hpp" + +/// @file +/// @brief Single source of truth for the ITCH 5.0 wire layout metadata shared by +/// the eager parser, the zero-copy overlay, and the encoder. +/// +/// Keeping the per-type wire size and the canonical list of message types in one +/// place means the dispatch table, the overlay accessors, and the writer can +/// never drift apart from one another. +/// +/// @author Bertin Balouki SIMYELI + +namespace itch::detail { + +/// @brief The on-wire ITCH timestamp is 48 bits (6 bytes), but every message +/// struct stores it in a 64-bit field, so each struct is exactly two +/// bytes wider than its encoding. The timestamp is the only field whose +/// storage width differs from its wire width. +constexpr std::size_t TIMESTAMP_STRUCT_PADDING = sizeof(std::uint64_t) - 6; + +/// @brief The exact on-wire size, in bytes, of a fully formed message of type T. +template +constexpr std::size_t WIRE_SIZE = sizeof(MsgType) - TIMESTAMP_STRUCT_PADDING; + +// Lock the padding assumption to the spec lengths so a future struct change that +// breaks the derivation is caught at compile time rather than at runtime. +static_assert(WIRE_SIZE == 12); +static_assert(WIRE_SIZE == 39); +static_assert(WIRE_SIZE == 36); +static_assert(WIRE_SIZE == 50); +static_assert(WIRE_SIZE == 48); + +/// @brief Invokes `visitor.template operator()(type_byte)` once for each +/// ITCH 5.0 message type, in a fixed order. +/// +/// This is the canonical registry of message types. Both the parser's dispatch +/// table and any other component that needs to enumerate the message set build +/// on it, so the set is defined exactly once. +/// +/// @tparam Visitor Callable type providing a templated `operator()(char)`. +/// @param visitor The visitor invoked once per registered message type. +template +constexpr auto for_each_message_type(Visitor&& visitor) -> void { + visitor.template operator()('S'); + visitor.template operator()('R'); + visitor.template operator()('H'); + visitor.template operator()('Y'); + visitor.template operator()('L'); + visitor.template operator()('V'); + visitor.template operator()('W'); + visitor.template operator()('K'); + visitor.template operator()('J'); + visitor.template operator()('h'); + visitor.template operator()('A'); + visitor.template operator()('F'); + visitor.template operator()('E'); + visitor.template operator()('C'); + visitor.template operator()('X'); + visitor.template operator()('D'); + visitor.template operator()('U'); + visitor.template operator()('P'); + visitor.template operator()('Q'); + visitor.template operator()('B'); + visitor.template operator()('I'); + visitor.template operator()('N'); + visitor.template operator()('O'); +} + +} // namespace itch::detail diff --git a/include/itch/encoder.hpp b/include/itch/encoder.hpp new file mode 100644 index 0000000..c01605c --- /dev/null +++ b/include/itch/encoder.hpp @@ -0,0 +1,35 @@ +#pragma once + +/// @file +/// @brief Serializes parsed ITCH messages back to valid wire bytes. +/// +/// The encoder is the inverse of the parser: it writes each message's fields in +/// the spec's order and network (big-endian) byte order, including the 48-bit +/// timestamp. It is used to synthesize valid ITCH streams for testing, scenario +/// generation, and golden fixtures, and it guarantees the round-trip property +/// `parse(encode(msg)) == msg` for every supported message type. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include + +#include "itch/messages.hpp" + +namespace itch { + +/// @brief Encodes a message body and header without the 2-byte length prefix. +/// +/// @param message The parsed message to encode. +/// @return The raw message bytes (type byte first), exactly as they appear inside +/// a frame. +[[nodiscard]] auto encode_message(const Message& message) -> std::vector; + +/// @brief Encodes a complete length-prefixed frame (2-byte big-endian length +/// followed by the message bytes), ready to concatenate into a stream. +/// +/// @param message The parsed message to encode. +/// @return The length-prefixed frame bytes, ready to append to a stream. +[[nodiscard]] auto encode_frame(const Message& message) -> std::vector; + +} // namespace itch diff --git a/include/itch/indicators.hpp b/include/itch/indicators.hpp index edb4e5a..3d17758 100644 --- a/include/itch/indicators.hpp +++ b/include/itch/indicators.hpp @@ -1,23 +1,97 @@ #pragma once -#include -#include +/// @file +/// @brief Compile-time lookup tables translating ITCH single- and +/// multi-character indicator codes into human-readable descriptions. +/// +/// These tables back the enum-like character fields scattered across the ITCH +/// message definitions (system events, market categories, financial status, +/// trading actions, and so on), giving callers a fast, allocation-free way to +/// render a wire code as descriptive text. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include #include +#include namespace itch { namespace indicators { -using sv = std::string_view; -const std::map SYSTEM_EVENT_CODES = { +/// @brief An immutable, compile-time lookup table from a key to a description. +/// +/// Replaces the namespace-scope `std::map` globals that this header used to +/// define: those incurred dynamic initialization in every translation unit and +/// risked one-definition-rule trouble. A `FrozenMap` is a literal type held in +/// read-only storage. Its entries are sorted once at compile time, so lookups +/// are a branch-light binary search with no allocation and no runtime setup. +/// +/// @tparam KeyType The key type (`char` for single-letter codes, or +/// `std::string_view` for multi-character codes). +/// @tparam Size The number of entries. +template +class FrozenMap { + public: + using value_type = std::pair; + + /// @brief Constructs the map from an unsorted list of key/description + /// entries, sorting them by key once so that `find` can binary + /// search. + /// + /// @param entries The key/description pairs backing this map, in any order. + constexpr explicit FrozenMap(std::array entries) noexcept + : m_entries {entries} { + std::ranges::sort(m_entries, {}, &value_type::first); + } + + /// @brief Returns the description for a key, or `std::nullopt` if absent. + /// + /// @param key The key to look up. + /// @return The matching description, or `std::nullopt` if no entry has this key. + [[nodiscard]] constexpr auto find(const KeyType& key + ) const noexcept -> std::optional { + const auto iter = std::ranges::lower_bound(m_entries, key, {}, &value_type::first); + if (iter != m_entries.end() && iter->first == key) { + return iter->second; + } + return std::nullopt; + } + + /// @brief Returns the description for a key, or a fallback if absent. + /// + /// @param key The key to look up. + /// @param fallback The description to return when `key` is not found. + /// @return The matching description, or `fallback` if no entry has this key. + [[nodiscard]] constexpr auto at_or(const KeyType& key, std::string_view fallback) const noexcept + -> std::string_view { + return find(key).value_or(fallback); + } + + private: + std::array m_entries; +}; + +/// @brief Deduces `FrozenMap`'s template arguments from a brace-initialized +/// array of key/description pairs. +/// +/// @tparam KeyType The key type (`char` for single-letter codes, or +/// `std::string_view` for multi-character codes). +/// @tparam Size The number of entries. +template +FrozenMap(std::array, Size>) -> FrozenMap; + +inline constexpr FrozenMap SYSTEM_EVENT_CODES {std::to_array>({ {'O', "Start of Messages"}, {'S', "Start of System hours"}, {'Q', "Start of Market hours"}, {'M', "End of Market hours"}, {'E', "End of System hours"}, {'C', "End of Messages"}, -}; +})}; -const std::map MARKET_CATEGORY = { +inline constexpr FrozenMap MARKET_CATEGORY {std::to_array>({ {'N', "NYSE"}, {'A', "AMEX"}, {'P', "Arca"}, @@ -27,180 +101,189 @@ const std::map MARKET_CATEGORY = { {'Z', "BATS"}, {'V', "Investors Exchange"}, {' ', "Not available"}, -}; +})}; -const std::map FINANCIAL_STATUS_INDICATOR = { - {'D', "Deficient"}, - {'E', "Delinquent"}, - {'Q', "Bankrupt"}, - {'S', "Suspended"}, - {'G', "Deficient and Bankrupt"}, - {'H', "Deficient and Delinquent"}, - {'J', "Delinquent and Bankrupt"}, - {'K', "Deficient, Delinquent and Bankrupt"}, - {'C', "Creations and/or Redemptions Suspended for Exchange Traded Product"}, - {'N', "Normal (Default): Issuer is NOT Deficient, Delinquent, or Bankrupt"}, - {' ', "Not available. Firms should refer to SIAC feeds for code if needed"}, +inline constexpr FrozenMap FINANCIAL_STATUS_INDICATOR { + std::to_array>({ + {'D', "Deficient"}, + {'E', "Delinquent"}, + {'Q', "Bankrupt"}, + {'S', "Suspended"}, + {'G', "Deficient and Bankrupt"}, + {'H', "Deficient and Delinquent"}, + {'J', "Delinquent and Bankrupt"}, + {'K', "Deficient, Delinquent and Bankrupt"}, + {'C', "Creations and/or Redemptions Suspended for Exchange Traded Product"}, + {'N', "Normal (Default): Issuer is NOT Deficient, Delinquent, or Bankrupt"}, + {' ', "Not available. Firms should refer to SIAC feeds for code if needed"}, + }) }; -const std::map ISSUE_CLASSIFICATION_VALUES = { - {'A', "American Depositary Share"}, - {'B', "Bond"}, - {'C', "Common Stock"}, - {'F', "Depository Receipt"}, - {'I', "144A"}, - {'L', "Limited Partnership"}, - {'N', "Notes"}, - {'O', "Ordinary Share"}, - {'P', "Preferred Stock"}, - {'Q', "Other Securities"}, - {'R', "Right"}, - {'S', "Shares of Beneficial Interest"}, - {'T', "Convertible Debenture"}, - {'U', "Unit"}, - {'V', "Units/Beneficial Interest"}, - {'W', "Warrant"}, +inline constexpr FrozenMap ISSUE_CLASSIFICATION_VALUES { + std::to_array>({ + {'A', "American Depositary Share"}, + {'B', "Bond"}, + {'C', "Common Stock"}, + {'F', "Depository Receipt"}, + {'I', "144A"}, + {'L', "Limited Partnership"}, + {'N', "Notes"}, + {'O', "Ordinary Share"}, + {'P', "Preferred Stock"}, + {'Q', "Other Securities"}, + {'R', "Right"}, + {'S', "Shares of Beneficial Interest"}, + {'T', "Convertible Debenture"}, + {'U', "Unit"}, + {'V', "Units/Beneficial Interest"}, + {'W', "Warrant"}, + }) }; -const std::map ISSUE_SUB_TYPE_VALUES = { - {"A", "Preferred Trust Securities"}, - {"AI", "Alpha Index ETNs"}, - {"B", "Index Based Derivative"}, - {"C", "Common Shares"}, - {"CB", "Commodity Based Trust Shares"}, - {"CF", "Commodity Futures Trust Shares"}, - {"CL", "Commodity-Linked Securities"}, - {"CM", "Commodity Index Trust Shares"}, - {"CO", "Collateralized Mortgage Obligation"}, - {"CT", "Currency Trust Shares"}, - {"CU", "Commodity-Currency-Linked Securities"}, - {"CW", "Currency Warrants"}, - {"D", "Global Depositary Shares"}, - {"E", "ETF-Portfolio Depositary Receipt"}, - {"EG", "Equity Gold Shares"}, - {"EI", "ETN-Equity Index-Linked Securities"}, - {"EM", "NextShares Exchange Traded Managed Fund*"}, - {"EN", "Exchange Traded Notes"}, - {"EU", "Equity Units"}, - {"F", "HOLDRS"}, - {"FI", "ETN-Fixed Income-Linked Securities"}, - {"FL", "ETN-Futures-Linked Securities"}, - {"G", "Global Shares"}, - {"I", "ETF-Index Fund Shares"}, - {"IR", "Interest Rate"}, - {"IW", "Index Warrant"}, - {"IX", "Index-Linked Exchangeable Notes"}, - {"J", "Corporate Backed Trust Security"}, - {"L", "Contingent Litigation Right"}, - {"LL", "Limited Liability Company (LLC)"}, - {"M", "Equity-Based Derivative"}, - {"MF", "Managed Fund Shares"}, - {"ML", "ETN-Multi-Factor Index-Linked Securities"}, - {"MT", "Managed Trust Securities"}, - {"N", "NY Registry Shares"}, - {"O", "Open Ended Mutual Fund"}, - {"P", "Privately Held Security"}, - {"PP", "Poison Pill"}, - {"PU", "Partnership Units"}, - {"Q", "Closed-End Funds"}, - {"R", "Reg-S"}, - {"RC", "Commodity-Redeemable Commodity-Linked Securities"}, - {"RF", "ETN-Redeemable Futures-Linked Securities"}, - {"RT", "REIT"}, - {"RU", "Commodity-Redeemable Currency-Linked Securities"}, - {"S", "SEED"}, - {"SC", "Spot Rate Closing"}, - {"SI", "Spot Rate Intraday"}, - {"T", "Tracking Stock"}, - {"TC", "Trust Certificates"}, - {"TU", "Trust Units"}, - {"U", "Portal"}, - {"V", "Contingent Value Right"}, - {"W", "Trust Issued Receipts"}, - {"WC", "World Currency Option"}, - {"X", "Trust"}, - {"Y", "Other"}, - {"Z", "Not Applicable"}, - +inline constexpr FrozenMap ISSUE_SUB_TYPE_VALUES { + std::to_array>({ + {"A", "Preferred Trust Securities"}, + {"AI", "Alpha Index ETNs"}, + {"B", "Index Based Derivative"}, + {"C", "Common Shares"}, + {"CB", "Commodity Based Trust Shares"}, + {"CF", "Commodity Futures Trust Shares"}, + {"CL", "Commodity-Linked Securities"}, + {"CM", "Commodity Index Trust Shares"}, + {"CO", "Collateralized Mortgage Obligation"}, + {"CT", "Currency Trust Shares"}, + {"CU", "Commodity-Currency-Linked Securities"}, + {"CW", "Currency Warrants"}, + {"D", "Global Depositary Shares"}, + {"E", "ETF-Portfolio Depositary Receipt"}, + {"EG", "Equity Gold Shares"}, + {"EI", "ETN-Equity Index-Linked Securities"}, + {"EM", "NextShares Exchange Traded Managed Fund*"}, + {"EN", "Exchange Traded Notes"}, + {"EU", "Equity Units"}, + {"F", "HOLDRS"}, + {"FI", "ETN-Fixed Income-Linked Securities"}, + {"FL", "ETN-Futures-Linked Securities"}, + {"G", "Global Shares"}, + {"I", "ETF-Index Fund Shares"}, + {"IR", "Interest Rate"}, + {"IW", "Index Warrant"}, + {"IX", "Index-Linked Exchangeable Notes"}, + {"J", "Corporate Backed Trust Security"}, + {"L", "Contingent Litigation Right"}, + {"LL", "Limited Liability Company (LLC)"}, + {"M", "Equity-Based Derivative"}, + {"MF", "Managed Fund Shares"}, + {"ML", "ETN-Multi-Factor Index-Linked Securities"}, + {"MT", "Managed Trust Securities"}, + {"N", "NY Registry Shares"}, + {"O", "Open Ended Mutual Fund"}, + {"P", "Privately Held Security"}, + {"PP", "Poison Pill"}, + {"PU", "Partnership Units"}, + {"Q", "Closed-End Funds"}, + {"R", "Reg-S"}, + {"RC", "Commodity-Redeemable Commodity-Linked Securities"}, + {"RF", "ETN-Redeemable Futures-Linked Securities"}, + {"RT", "REIT"}, + {"RU", "Commodity-Redeemable Currency-Linked Securities"}, + {"S", "SEED"}, + {"SC", "Spot Rate Closing"}, + {"SI", "Spot Rate Intraday"}, + {"T", "Tracking Stock"}, + {"TC", "Trust Certificates"}, + {"TU", "Trust Units"}, + {"U", "Portal"}, + {"V", "Contingent Value Right"}, + {"W", "Trust Issued Receipts"}, + {"WC", "World Currency Option"}, + {"X", "Trust"}, + {"Y", "Other"}, + {"Z", "Not Applicable"}, + }) }; -const std::map TRADING_ACTION_REASON_CODES = { - {"T1", "Halt News Pending"}, - {"T2", "Halt News Disseminated"}, - {"T3", "News and Resumption Times"}, - {"T5", "Single Security Trading Pause In Effect"}, - {"T6", "Regulatory Halt - Extraordinary Market Activity"}, - {"T7", "Single Security Trading Pause / Quotation Only Period"}, - {"T8", "Halt ETF"}, - {"T12", "Trading Halted; For Information Requested by Listing Market"}, - {"H4", "Halt Non-Compliance"}, - {"H9", "Halt Filings Not Current"}, - {"H10", "Halt SEC Trading Suspension"}, - {"H11", "Halt Regulatory Concern"}, - {"O1", "Operations Halt; Contact Market Operations"}, - {"LUDP", "Volatility Trading Pause"}, - {"LUDS", "Volatility Trading Pause - Straddle Condition"}, - {"MWC0", "Market Wide Circuit Breaker Halt - Carry over from previous day"}, - {"MWC1", "Market Wide Circuit Breaker Halt - Level 1"}, - {"MWC2", "Market Wide Circuit Breaker Halt - Level 2"}, - {"MWC3", "Market Wide Circuit Breaker Halt - Level 3"}, - {"MWCQ", "Market Wide Circuit Breaker Resumption"}, - {"IPO1", "IPO Issue Not Yet Trading"}, - {"IPOQ", "IPO Security Released for Quotation (Nasdaq Securities Only)"}, - {"IPOE", "IPO Security — Positioning Window Extension (Nasdaq Securities Only)"}, - {"M1", "Corporate Action"}, - {"M2", "Quotation Not Available"}, - {"R1", "New Issue Available"}, - {"R2", "Issue Available"}, - {"R4", "Qualifications Issues Reviewed/Resolved; Quotations/Trading to Resume"}, - {"R9", "Filing Requirements Satisfied/Resolved; Quotations/Trading To Resume"}, - {"C3", "Issuer News Not Forthcoming; Quotations/Trading To Resume"}, - {"C4", "Qualifications Halt Ended; Maintenance Requirements Met; Resume"}, - {"C9", "Qualifications Halt Concluded; Filings Met; Quotes/Trades To Resume"}, - {"C11", - "Trade Halt Concluded By Other Regulatory Authority; Quotes/Trades " - "Resume"}, - {" ", "Reason Not Available"}, +inline constexpr FrozenMap TRADING_ACTION_REASON_CODES { + std::to_array>({ + {"T1", "Halt News Pending"}, + {"T2", "Halt News Disseminated"}, + {"T3", "News and Resumption Times"}, + {"T5", "Single Security Trading Pause In Effect"}, + {"T6", "Regulatory Halt - Extraordinary Market Activity"}, + {"T7", "Single Security Trading Pause / Quotation Only Period"}, + {"T8", "Halt ETF"}, + {"T12", "Trading Halted; For Information Requested by Listing Market"}, + {"H4", "Halt Non-Compliance"}, + {"H9", "Halt Filings Not Current"}, + {"H10", "Halt SEC Trading Suspension"}, + {"H11", "Halt Regulatory Concern"}, + {"O1", "Operations Halt; Contact Market Operations"}, + {"LUDP", "Volatility Trading Pause"}, + {"LUDS", "Volatility Trading Pause - Straddle Condition"}, + {"MWC0", "Market Wide Circuit Breaker Halt - Carry over from previous day"}, + {"MWC1", "Market Wide Circuit Breaker Halt - Level 1"}, + {"MWC2", "Market Wide Circuit Breaker Halt - Level 2"}, + {"MWC3", "Market Wide Circuit Breaker Halt - Level 3"}, + {"MWCQ", "Market Wide Circuit Breaker Resumption"}, + {"IPO1", "IPO Issue Not Yet Trading"}, + {"IPOQ", "IPO Security Released for Quotation (Nasdaq Securities Only)"}, + {"IPOE", "IPO Security - Positioning Window Extension (Nasdaq Securities Only)"}, + {"M1", "Corporate Action"}, + {"M2", "Quotation Not Available"}, + {"R1", "New Issue Available"}, + {"R2", "Issue Available"}, + {"R4", "Qualifications Issues Reviewed/Resolved; Quotations/Trading to Resume"}, + {"R9", "Filing Requirements Satisfied/Resolved; Quotations/Trading To Resume"}, + {"C3", "Issuer News Not Forthcoming; Quotations/Trading To Resume"}, + {"C4", "Qualifications Halt Ended; Maintenance Requirements Met; Resume"}, + {"C9", "Qualifications Halt Concluded; Filings Met; Quotes/Trades To Resume"}, + {"C11", "Trade Halt Concluded By Other Regulatory Authority; Quotes/Trades Resume"}, + {" ", "Reason Not Available"}, + }) }; -const std::map TRADING_STATES = { +inline constexpr FrozenMap TRADING_STATES {std::to_array>({ {'H', "Halted across all U.S. equity markets / SROs"}, {'P', "Paused across all U.S. equity markets / SROs"}, {'Q', "Quotation only period for cross-SRO halt or pause"}, {'T', "Trading on NASDAQ"}, -}; +})}; -const std::map MARKET_MAKER_MODE = { +inline constexpr FrozenMap MARKET_MAKER_MODE {std::to_array>({ {'N', "Normal"}, {'P', "Passive"}, {'S', "Syndicate"}, {'R', "Pre-syndicate"}, {'L', "Penalty"}, -}; +})}; -const std::map MARKET_PARTICIPANT_STATE = { - {'A', "Active"}, - {'E', "Excused"}, - {'W', "Withdrawn"}, - {'S', "Suspended"}, - {'D', "Deleted"}, +inline constexpr FrozenMap MARKET_PARTICIPANT_STATE { + std::to_array>({ + {'A', "Active"}, + {'E', "Excused"}, + {'W', "Withdrawn"}, + {'S', "Suspended"}, + {'D', "Deleted"}, + }) }; -const std::map PRICE_VARIATION_INDICATOR = { - {'L', "Less than 1%"}, - {'1', "1 to 1.99%"}, - {'2', "2 to 2.99%"}, - {'3', "3 to 3.99%"}, - {'4', "4 to 4.99%"}, - {'5', "5 to 5.99%"}, - {'6', "6 to 6.99%"}, - {'7', "7 to 7.99%"}, - {'8', "8 to 8.99%"}, - {'9', "9 to 9.99%"}, - {'A', "10 to 19.99%"}, - {'B', "20 to 29.99%"}, - {'C', "30% or greater"}, - {' ', "Cannot be calculated"}, +inline constexpr FrozenMap PRICE_VARIATION_INDICATOR { + std::to_array>({ + {'L', "Less than 1%"}, + {'1', "1 to 1.99%"}, + {'2', "2 to 2.99%"}, + {'3', "3 to 3.99%"}, + {'4', "4 to 4.99%"}, + {'5', "5 to 5.99%"}, + {'6', "6 to 6.99%"}, + {'7', "7 to 7.99%"}, + {'8', "8 to 8.99%"}, + {'9', "9 to 9.99%"}, + {'A', "10 to 19.99%"}, + {'B', "20 to 29.99%"}, + {'C', "30% or greater"}, + {' ', "Cannot be calculated"}, + }) }; } // namespace indicators diff --git a/include/itch/io/arrow_export.hpp b/include/itch/io/arrow_export.hpp new file mode 100644 index 0000000..3082f41 --- /dev/null +++ b/include/itch/io/arrow_export.hpp @@ -0,0 +1,87 @@ +#pragma once + +/// @file +/// @brief Apache Arrow / Parquet columnar export. +/// +/// This header is only meaningful when the library is built with +/// `-DITCH_WITH_ARROW=ON` (Arrow and Parquet provided via vcpkg). When the option +/// is off the class is not compiled, keeping the core dependency-free. The export +/// turns a feed into a columnar table that drops straight into pandas, Polars, +/// DuckDB, and Spark pipelines. +/// +/// @author Bertin Balouki SIMYELI + +#ifdef ITCH_WITH_ARROW + +#include +#include +#include + +#include "itch/messages.hpp" + +namespace arrow { +class Array; +} // namespace arrow + +namespace itch::io { + +/// @brief Accumulates parsed messages into Arrow columns and writes Parquet. +/// +/// Messages are flattened into the same normalized wide schema as `CsvSink`: +/// `message_type, timestamp, stock_locate, tracking_number, symbol,` +/// `reference_number, side, shares, price, match_number, printable, extra`. +/// Append each message with `append`, then call `write_parquet` to flush a file. +class ArrowExporter { + public: + /// @brief Constructs an empty exporter with no rows appended. + ArrowExporter(); + + /// @brief Destroys the exporter, releasing any held Arrow/Parquet resources. + ~ArrowExporter(); + + /// @brief Non-copyable; move-only. + ArrowExporter(const ArrowExporter&) = delete; + + /// @brief Non-copyable; move-only. + /// + /// @return Reference to this exporter. + auto operator=(const ArrowExporter&) -> ArrowExporter& = delete; + + /// @brief Move-constructs an exporter, transferring ownership of its state. + ArrowExporter(ArrowExporter&&) noexcept = default; + + /// @brief Move-assigns an exporter, transferring ownership of its state. + /// + /// @return Reference to this exporter. + auto operator=(ArrowExporter&&) noexcept -> ArrowExporter& = default; + + /// @brief Appends one message as a row to the column builders. + /// + /// @param message The parsed message to append. + auto append(const Message& message) -> void; + + /// @brief The number of rows appended so far. + /// + /// @return The count of rows appended since construction. + [[nodiscard]] auto rows() const noexcept -> std::uint64_t; + + /// @brief Finishes the builders and writes a Parquet file to `path`. + /// + /// @param path Filesystem path to write the Parquet file to. + /// @return True on success; on failure `error()` holds a description. + [[nodiscard]] auto write_parquet(const std::string& path) -> bool; + + /// @brief A description of the last failure, or empty on success. + /// + /// @return The last failure message, or an empty string if the last operation + /// succeeded. + [[nodiscard]] auto error() const -> const std::string&; + + private: + struct Impl; + std::unique_ptr m_impl; +}; + +} // namespace itch::io + +#endif // ITCH_WITH_ARROW diff --git a/include/itch/io/csv_sink.hpp b/include/itch/io/csv_sink.hpp new file mode 100644 index 0000000..a5dd90b --- /dev/null +++ b/include/itch/io/csv_sink.hpp @@ -0,0 +1,57 @@ +#pragma once + +/// @file +/// @brief CSV output sink for parsed ITCH messages. +/// +/// `CsvSink` is the dependency-free, universal output format: it flattens the +/// heterogeneous ITCH message set into one wide, normalized row schema suitable +/// for quick inspection and legacy tooling. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include + +#include "itch/io/sink.hpp" +#include "itch/messages.hpp" + +namespace itch::io { + +/// @brief Writes parsed ITCH messages to a CSV stream as a flat, normalized table. +/// +/// ITCH messages are heterogeneous, so the sink flattens them into one wide row +/// schema sharing the common header plus the most frequently used per-type fields. +/// Fields a given message does not carry are left blank. This is the universal, +/// dependency-free output for quick inspection and legacy tooling; for columnar +/// analytics use the Arrow/Parquet exporter instead. +/// +/// Columns: `message_type,timestamp,stock_locate,tracking_number,symbol,` +/// `reference_number,side,shares,price,match_number,printable,extra`. +class CsvSink : public MessageSink { + public: + /// @brief Constructs a sink writing to `out`, emitting the header row unless + /// `write_header` is false (useful when appending to an existing file). + /// + /// @param out Output stream the CSV rows are written to. + /// @param write_header Whether to emit the CSV header row on construction. + explicit CsvSink(std::ostream& out, bool write_header = true); + + /// @brief Writes one message as a CSV row. + /// + /// @param message The parsed message to write. + auto write(const Message& message) -> void override; + + /// @brief Flushes the underlying stream. + auto flush() -> void override; + + /// @brief The number of rows written so far. + /// + /// @return The count of rows written since construction. + [[nodiscard]] auto rows_written() const noexcept -> std::uint64_t { return m_rows_written; } + + private: + std::ostream& m_out; + std::uint64_t m_rows_written {0}; +}; + +} // namespace itch::io diff --git a/include/itch/io/sink.hpp b/include/itch/io/sink.hpp new file mode 100644 index 0000000..efa9125 --- /dev/null +++ b/include/itch/io/sink.hpp @@ -0,0 +1,54 @@ +#pragma once + +/// @file +/// @brief Abstract streaming sink interface for parsed ITCH messages. +/// +/// `MessageSink` is the common output interface implemented by `CsvSink`, +/// `ArrowExporter`, and other consumers that receive one message at a time from +/// a parse callback. +/// +/// @author Bertin Balouki SIMYELI + +#include "itch/messages.hpp" + +namespace itch::io { + +/// @brief A generic streaming sink for parsed ITCH messages. +/// +/// Sinks are the universal output side of the library: a parse callback can hand +/// every message to a sink, which writes it somewhere (CSV, Arrow, a socket, a +/// counter). Implementations override `write`; `flush` is optional. +class MessageSink { + public: + /// @brief Default-constructs an empty sink. + MessageSink() = default; + + /// @brief Copy-constructs a sink. + MessageSink(const MessageSink&) = default; + + /// @brief Move-constructs a sink. + MessageSink(MessageSink&&) noexcept = default; + + /// @brief Copy-assigns a sink. + /// + /// @return Reference to this sink. + auto operator=(const MessageSink&) -> MessageSink& = default; + + /// @brief Move-assigns a sink. + /// + /// @return Reference to this sink. + auto operator=(MessageSink&&) noexcept -> MessageSink& = default; + + /// @brief Virtual destructor for safe polymorphic destruction. + virtual ~MessageSink() = default; + + /// @brief Consumes one parsed message. + /// + /// @param message The parsed message to write. + virtual auto write(const Message& message) -> void = 0; + + /// @brief Flushes any buffered output. The default is a no-op. + virtual auto flush() -> void {} +}; + +} // namespace itch::io diff --git a/include/itch/messages.hpp b/include/itch/messages.hpp index 826d82b..ec0d92c 100644 --- a/include/itch/messages.hpp +++ b/include/itch/messages.hpp @@ -1,5 +1,18 @@ #pragma once +/// @file +/// @brief Defines every ITCH 5.0 message struct, the `Message` variant, and the +/// stream-printing helpers used to format them. +/// +/// The TotalView ITCH feed is composed of a series of messages that describe +/// orders added to, removed from, and executed on Nasdaq, as well as Cross and +/// Stock Directory information. Each message begins with a one-byte Message Type +/// field that identifies the structure of the remainder of the message; this +/// header defines one struct per message type, the `Message` variant able to hold +/// any of them, and free functions to print a message to a stream. +/// +/// @author Bertin Balouki SIMYELI + #include #include #include @@ -11,284 +24,429 @@ namespace itch { // DISABLE PADDING #pragma pack(push, 1) +/// @brief System Event (`S`): signals a market or data-feed handler event. +/// +/// Used to signal a market or data feed handler event. The `event_code` marks the +/// major points of the trading day: start and end of the messages stream, of +/// system hours, and of market hours. Consumers use it to bracket a session and +/// to know when the book is expected to be live. struct SystemEventMessage { - char message_type = 'S'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char event_code; + char message_type = 'S'; ///< Always 'S'. + uint16_t stock_locate; ///< Locate code; 0 for system-wide events. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char event_code; ///< 'O','S','Q','M','E','C' (see indicators::SYSTEM_EVENT_CODES). }; +/// @brief Stock Directory (`R`): the trading and listing reference data for a +/// security, disseminated at the start of each trading day. +/// +/// Disseminated for all active Nasdaq-traded symbols at the start of each trading +/// day. Market-data redistributors use it to populate fields such as the +/// Financial Status Indicator and Market Category so each security is classified +/// and displayed correctly. A security not named in any Stock Directory message +/// should not appear in later messages for that day. struct StockDirectoryMessage { - char message_type = 'R'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char market_category; - char financial_status_indicator; - uint32_t round_lot_size; - char round_lots_only; - char issue_classification; - char issue_sub_type[2]; - char authenticity; - char short_sale_threshold_indicator; - char ipo_flag; - char luld_ref; - char etp_flag; - uint32_t etp_leverage_factor; - char inverse_indicator; + char message_type = 'R'; ///< Always 'R'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char market_category; ///< Listing market (see indicators::MARKET_CATEGORY). + char financial_status_indicator; ///< Financial status of the issuer. + uint32_t round_lot_size; ///< Number of shares in a round lot. + char round_lots_only; ///< 'Y' if only round lots may be entered, else 'N'. + char issue_classification; ///< Security class (see indicators::ISSUE_CLASSIFICATION_VALUES). + char issue_sub_type[2]; ///< Security sub-type (see indicators::ISSUE_SUB_TYPE_VALUES). + char authenticity; ///< 'P' production / 'T' test security. + char short_sale_threshold_indicator; ///< Reg SHO threshold: 'Y','N',' '. + char ipo_flag; ///< 'Y' if a new IPO, 'N' if not, ' ' if not available. + char luld_ref; ///< LULD reference price tier. + char etp_flag; ///< 'Y' if an exchange-traded product, else 'N'. + uint32_t etp_leverage_factor; ///< Leverage factor of the ETP (if applicable). + char inverse_indicator; ///< 'Y' if the ETP is an inverse product. }; +/// @brief Stock Trading Action (`H`): a change in the trading status of a +/// security (halted, paused, quotation-only, or trading). +/// +/// An administrative message conveying the current trading status of a security. +/// It is sent before the open and again whenever the status changes, such as a +/// halt, a pause, a release for quotation, or a resumption of trading. The +/// `reason` code explains why the action occurred. struct StockTradingActionMessage { - char message_type = 'H'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char trading_state; - char reserved; - char reason[4]; + char message_type = 'H'; ///< Always 'H'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char trading_state; ///< 'H','P','Q','T' (see indicators::TRADING_STATES). + char reserved; ///< Reserved. + char reason[4]; ///< Trading-action reason code (see indicators::TRADING_ACTION_REASON_CODES). }; +/// @brief Reg SHO Short Sale Price Test Restriction (`Y`): the Reg SHO short-sale +/// restriction state for a security. +/// +/// Informs recipients of the SEC Rule 201 (Regulation SHO) short-sale price-test +/// restriction status for a security. It is sent both as a pre-open spin for every +/// security and intraday whenever the restriction status changes. struct RegSHOMessage { - char message_type = 'Y'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char reg_sho_action; + char message_type = 'Y'; ///< Always 'Y'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char reg_sho_action; ///< '0' no restriction, '1' restriction in effect, '2' remains. }; +/// @brief Market Participant Position (`L`): a market participant's (market +/// maker's) status in a security. +/// +/// Disseminated at the start of the trading day and whenever a status changes. It +/// provides, per firm registered in an issue, the Primary Market Maker status, the +/// Market Maker mode, and the Market Participant state. struct MarketParticipantPositionMessage { - char message_type = 'L'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char mpid[4]; - char stock[8]; - char primary_market_maker; - char market_maker_mode; - char market_participant_state; + char message_type = 'L'; ///< Always 'L'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char mpid[4]; ///< Market participant identifier. + char stock[8]; ///< Stock symbol, right padded with spaces. + char primary_market_maker; ///< 'Y' if the primary market maker, else 'N'. + char market_maker_mode; ///< Quotation mode (see indicators::MARKET_MAKER_MODE). + char market_participant_state; ///< State (see indicators::MARKET_PARTICIPANT_STATE). }; +/// @brief MWCB Decline Level (`V`): the Market-Wide Circuit Breaker breach points +/// for the day. +/// +/// Informs recipients what the daily Market-Wide Circuit Breaker breach points are +/// set to for the current trading day. The three levels correspond to the 5%, 13%, +/// and 20% S&P 500 decline thresholds that trigger a market-wide trading halt. +/// +/// @note Unlike every other price field (4 implied decimals), the three level +/// prices here carry 8 implied decimals; use `MwcbPrice` to interpret them. struct MWCBDeclineLevelMessage { - char message_type = 'V'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t level1; - uint64_t level2; - uint64_t level3; + char message_type = 'V'; ///< Always 'V'. + uint16_t stock_locate; ///< Locate code; 0 for this market-wide message. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t level1; ///< Level 1 (5%) breach value, 8 implied decimals. + uint64_t level2; ///< Level 2 (13%) breach value, 8 implied decimals. + uint64_t level3; ///< Level 3 (20%) breach value, 8 implied decimals. }; +/// @brief MWCB Status (`W`): notification that a Market-Wide Circuit Breaker level +/// has been breached. +/// +/// Informs recipients when a Market-Wide Circuit Breaker has breached one of the +/// established decline levels, which triggers the corresponding market-wide halt. struct MWCBStatusMessage { - char message_type = 'W'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char breached_level; + char message_type = 'W'; ///< Always 'W'. + uint16_t stock_locate; ///< Locate code; 0 for this market-wide message. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char breached_level; ///< '1', '2', or '3' for the level breached. }; +/// @brief IPO Quoting Period Update (`K`): the anticipated IPO quotation release +/// time and price for a security. +/// +/// Indicates the anticipated IPO quotation release time for a security. A +/// cancellation or postponement of the IPO is signalled by setting both the +/// release time and the price to zero. struct IPOQuotingPeriodUpdateMessage { - char message_type = 'K'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - uint32_t ipo_quotation_release_time; - char ipo_quotation_release_qualifier; - uint32_t ipo_price; + char message_type = 'K'; ///< Always 'K'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t ipo_quotation_release_time; ///< Seconds past midnight of the anticipated release. + char ipo_quotation_release_qualifier; ///< 'A' anticipated, 'C' cancelled/postponed. + uint32_t ipo_price; ///< IPO price (4 implied decimals). }; +/// @brief LULD Auction Collar (`J`): the auction collar thresholds for a security +/// in a Limit-Up Limit-Down trading pause. +/// +/// Indicates the auction collar thresholds within which a paused security may +/// reopen following a Limit-Up Limit-Down (LULD) trading pause. struct LULDAuctionCollarMessage { - char message_type = 'J'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - uint32_t auction_collar_reference_price; - uint32_t upper_auction_collar_price; - uint32_t lower_auction_collar_price; - uint32_t auction_collar_extension; + char message_type = 'J'; ///< Always 'J'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t auction_collar_reference_price; ///< Reference price for the collars (4 decimals). + uint32_t upper_auction_collar_price; ///< Upper auction collar price (4 decimals). + uint32_t lower_auction_collar_price; ///< Lower auction collar price (4 decimals). + uint32_t auction_collar_extension; ///< Number of collar extensions so far. }; +/// @brief Operational Halt (`h`): an operational halt or resumption for a security +/// on a specific market center. +/// +/// Indicates the operational status of a security: a service interruption that +/// affects only the designated market center. This differs from a Stock Trading +/// Action: an operational halt is a venue-level interruption rather than a +/// regulatory or volatility trading halt. struct OperationalHaltMessage { - char message_type = 'h'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char market_code; - char operational_halt_action; + char message_type = 'h'; ///< Always 'h'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char market_code; ///< 'Q' Nasdaq, 'B' BX, 'X' PSX. + char operational_halt_action; ///< 'H' halted, 'T' resumed. }; +/// @brief Add Order, No MPID Attribution (`A`): a new displayable order has been +/// accepted and placed on the book. +/// +/// Indicates that Nasdaq has accepted a new, unattributed order and added it to +/// the displayable book with a day-unique order reference number. That reference +/// number is used by every subsequent execute, cancel, delete, and replace +/// message that acts on the order. struct AddOrderMessage { - char message_type = 'A'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - char buy_sell_indicator; - uint32_t shares; - char stock[8]; - uint32_t price; + char message_type = 'A'; ///< Always 'A'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Day-unique reference number for the order. + char buy_sell_indicator; ///< 'B' buy, 'S' sell. + uint32_t shares; ///< Displayed share quantity. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t price; ///< Display price (4 implied decimals). }; +/// @brief Add Order, With MPID Attribution (`F`): like Add Order, but attributed +/// to a market participant. +/// +/// Indicates that Nasdaq has accepted a new attributed order or quotation and +/// added it to the displayable book. It is identical to an Add Order message +/// except that it also carries the attributing market participant identifier. struct AddOrderMPIDAttributionMessage { - char message_type = 'F'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - char buy_sell_indicator; - uint32_t shares; - char stock[8]; - uint32_t price; - char attribution[4]; + char message_type = 'F'; ///< Always 'F'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Day-unique reference number for the order. + char buy_sell_indicator; ///< 'B' buy, 'S' sell. + uint32_t shares; ///< Displayed share quantity. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t price; ///< Display price (4 implied decimals). + char attribution[4]; ///< Market participant identifier (MPID). }; +/// @brief Order Executed (`E`): a resting order was executed in whole or in part +/// at its display price. +/// +/// Sent when an order on the book is executed in whole or in part at its display +/// price. Several of these may be sent for the same order reference number; their +/// effects are cumulative, and the order is removed from the book once it is fully +/// executed. struct OrderExecutedMessage { - char message_type = 'E'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - uint32_t executed_shares; - uint64_t match_number; + char message_type = 'E'; ///< Always 'E'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the executed order. + uint32_t executed_shares; ///< Number of shares executed. + uint64_t match_number; ///< Day-unique match number for the execution. }; +/// @brief Order Executed With Price (`C`): a resting order was executed at a price +/// different from its display price (and may be non-printable). +/// +/// Sent when an order on the book executes at a price different from its initial +/// display price, so the execution price is carried explicitly. The execution may +/// be marked non-printable, in which case it is excluded from the time-and-sales +/// tape (typically to be printed later in aggregate). struct OrderExecutedWithPriceMessage { - char message_type = 'C'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - uint32_t executed_shares; - uint64_t match_number; - char printable; - uint32_t execution_price; + char message_type = 'C'; ///< Always 'C'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the executed order. + uint32_t executed_shares; ///< Number of shares executed. + uint64_t match_number; ///< Day-unique match number for the execution. + char printable; ///< 'Y' if the trade is printable to the tape, else 'N'. + uint32_t execution_price; ///< Price at which the order executed (4 decimals). }; +/// @brief Order Cancel (`X`): a partial cancellation reduced the shares of a +/// resting order. +/// +/// Sent when an order on the book is modified by a partial cancellation: the +/// specified number of shares is removed from the order's display size while the +/// order itself remains on the book. A full cancellation is conveyed by an Order +/// Delete (`D`) message instead. struct OrderCancelMessage { - char message_type = 'X'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - uint32_t cancelled_shares; + char message_type = 'X'; ///< Always 'X'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the cancelled order. + uint32_t cancelled_shares; ///< Number of shares cancelled. }; +/// @brief Order Delete (`D`): a resting order was cancelled in its entirety and +/// removed from the book. +/// +/// Sent when an order on the book is cancelled in full. All remaining shares +/// become inaccessible and the order must be removed from the book. struct OrderDeleteMessage { - char message_type = 'D'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; + char message_type = 'D'; ///< Always 'D'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the deleted order. }; +/// @brief Order Replace (`U`): a resting order was replaced with a new order at a +/// new reference number, size, and/or price (same side and security). +/// +/// Sent when an order on the book is cancel-replaced. The original order's shares +/// become inaccessible, and the replacement is assigned a new reference number +/// that is used for all subsequent updates. The side and security are unchanged; +/// only the price and size may differ. struct OrderReplaceMessage { - char message_type = 'U'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t original_order_reference_number; - uint64_t new_order_reference_number; - uint32_t shares; - uint32_t price; + char message_type = 'U'; ///< Always 'U'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t original_order_reference_number; ///< Reference number being replaced. + uint64_t new_order_reference_number; ///< New reference number for the order. + uint32_t shares; ///< New displayed share quantity. + uint32_t price; ///< New display price (4 implied decimals). }; +/// @brief Trade, Non-Cross (`P`): an execution of a non-displayable order. It does +/// not affect the visible book but is disseminated as a print. +/// +/// Provides execution details for normal match events involving non-displayable +/// order types, transmitted when such an order executes in whole or in part. +/// Because the order was never on the displayable book, this message does not +/// change book state; it exists so that consumers can include these executions in +/// the trade tape and volume calculations. struct NonCrossTradeMessage { - char message_type = 'P'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t order_reference_number; - char buy_sell_indicator; - uint32_t shares; - char stock[8]; - uint32_t price; - uint64_t match_number; + char message_type = 'P'; ///< Always 'P'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t order_reference_number; ///< Reference number of the non-displayed order. + char buy_sell_indicator; ///< 'B' buy, 'S' sell. + uint32_t shares; ///< Number of shares traded. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t price; ///< Trade price (4 implied decimals). + uint64_t match_number; ///< Day-unique match number for the trade. }; +/// @brief Cross Trade (`Q`): the result of a cross (opening, closing, halt/IPO +/// cross) for a security. +/// +/// Indicates that Nasdaq has completed the cross process for a security. It is +/// sent following the Opening Cross, the Closing Cross, and Extended Market Close +/// (EMC) cross events, and reports the matched volume and the single cross price. struct CrossTradeMessage { - char message_type = 'Q'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t shares; - char stock[8]; - uint32_t cross_price; - uint64_t match_number; - char cross_type; + char message_type = 'Q'; ///< Always 'Q'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t shares; ///< Number of shares matched in the cross. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t cross_price; ///< Price at which the cross executed (4 decimals). + uint64_t match_number; ///< Day-unique match number for the cross. + char cross_type; ///< 'O' open, 'C' close, 'H' halt/IPO, 'I' intraday. }; +/// @brief Broken Trade (`B`): a previously disseminated execution has been broken +/// and should be removed from any trade record. +/// +/// Sent whenever an execution on Nasdaq is broken under the clearly-erroneous +/// execution policy. A trade break is final and cannot be reinstated; consumers +/// should remove the referenced match number from their trade records. struct BrokenTradeMessage { - char message_type = 'B'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t match_number; + char message_type = 'B'; ///< Always 'B'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t match_number; ///< Match number of the execution being broken. }; +/// @brief Net Order Imbalance Indicator (`I`): the imbalance and price-discovery +/// data disseminated during the opening and closing crosses. +/// +/// Disseminates Net Order Imbalance Indicator data at regular intervals during the +/// cross sessions. It reports the paired and imbalanced share quantities and the +/// hypothetical clearing prices (far, near, and current reference price) the cross +/// would produce, so participants can react before the cross executes. struct NOIIMessage { - char message_type = 'I'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - uint64_t paired_shares; - uint64_t imbalance_shares; - char imbalance_direction; - char stock[8]; - uint32_t far_price; - uint32_t near_price; - uint32_t current_reference_price; - char cross_type; - char price_variation_indicator; + char message_type = 'I'; ///< Always 'I'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + uint64_t paired_shares; ///< Shares paired at the current reference price. + uint64_t imbalance_shares; ///< Shares not paired (the imbalance). + char imbalance_direction; ///< 'B' buy, 'S' sell, 'N' none, 'O' insufficient, 'P' paused. + char stock[8]; ///< Stock symbol, right padded with spaces. + uint32_t far_price; ///< Cross price using only eligible interest (4 decimals). + uint32_t near_price; ///< Cross price using all interest (4 decimals). + uint32_t current_reference_price; ///< Price the cross would occur at now (4 decimals). + char cross_type; ///< 'O' open, 'C' close, 'H' halt/IPO cross. + char + price_variation_indicator; ///< Variation band (see indicators::PRICE_VARIATION_INDICATOR). }; +/// @brief Retail Price Improvement Indicator (`N`): the presence of retail price +/// improvement interest on the bid, the offer, or both. +/// +/// Identifies the presence of a retail price improvement interest indication (on +/// the bid, the offer, both, or none) for a Nasdaq-listed security. It signals +/// available retail liquidity without disclosing its size or price. struct RetailPriceImprovementIndicatorMessage { - char message_type = 'N'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char interest_flag; + char message_type = 'N'; ///< Always 'N'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char interest_flag; ///< 'B' bid, 'A' ask, 'C' both, 'N' none. }; +/// @brief Direct Listing with Capital Raise Price Discovery (`O`): price-discovery +/// data for a Direct Listing with a Capital Raise (DLCR) security. +/// +/// Disseminated only for Direct Listing with Capital Raise (DLCR) securities, once +/// the security passes its volatility test. It provides the allowable price +/// thresholds, the price range collars, and the anticipated execution price and +/// time used during the DLCR opening. struct DLCRMessage { - char message_type = 'O'; - uint16_t stock_locate; - uint16_t tracking_number; - uint64_t timestamp; - char stock[8]; - char open_eligibility_status; - uint32_t minimum_allowable_price; - uint32_t maximum_allowable_price; - uint32_t near_execution_price; - uint64_t near_execution_time; - uint32_t lower_price_range_collar; - uint32_t upper_price_range_collar; + char message_type = 'O'; ///< Always 'O'. + uint16_t stock_locate; ///< Locate code identifying the security. + uint16_t tracking_number; ///< Nasdaq internal tracking number. + uint64_t timestamp; ///< Nanoseconds past midnight. + char stock[8]; ///< Stock symbol, right padded with spaces. + char open_eligibility_status; ///< Whether the security is eligible to open. + uint32_t minimum_allowable_price; ///< Lowest allowable cross price (4 decimals). + uint32_t maximum_allowable_price; ///< Highest allowable cross price (4 decimals). + uint32_t near_execution_price; ///< Anticipated cross price (4 decimals). + uint64_t near_execution_time; ///< Time of the anticipated cross (ns past midnight). + uint32_t lower_price_range_collar; ///< Lower price range collar (4 decimals). + uint32_t upper_price_range_collar; ///< Upper price range collar (4 decimals). }; // RESTORE DEFAULT PADDING #pragma pack(pop) -/// The TotalView ITCH feed is composed of a series of messages that describe -/// orders added to, removed from, and executed on Nasdaq as well as disseminate -/// Cross and Stock Directory information. -/// Each message begins with a one-byte Message Type field that identifies the -/// structure of the remainder of the message. The Message Type is followed by -/// fields that are specific to each message type. -/// -/// All Message have the following attributes: -/// - message_type: A single letter that identify the message -/// - timestamp: Time at which the message was generated (Nanoseconds past -/// midnight) -/// - stock_locate: Locate code identifying the security -/// - tracking_number: Nasdaq internal tracking number -/// -/// for more details on each message type, see the +/// @brief A variant able to hold any one of the ITCH 5.0 message structs. +/// +/// The Message Type byte at the start of each frame identifies which struct is +/// active. All alternatives share four common attributes: `message_type` (a +/// single letter identifying the message), `timestamp` (nanoseconds past +/// midnight), `stock_locate` (locate code identifying the security), and +/// `tracking_number` (Nasdaq internal tracking number). For more details on each +/// message type, see the /// [documentation](https://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NQTVITCHspecification.pdf). /// /// @note @@ -328,15 +486,30 @@ constexpr int STOCK_LEN = 8; constexpr double PRICE_DIVISOR = 10000.0; constexpr double MWCB_PRICE_DIVISOR = 1.0E8; -// Convert char arrays to strings, trimming trailing spaces. -inline auto to_string(const char* arr, size_t size) -> std::string { +/// @brief Converts a fixed-width character array to a string, trimming trailing +/// spaces and NUL characters. +/// +/// @param source Pointer to the first character of the fixed-width field. +/// @param size Number of characters in the field. +/// @return The trimmed string, with trailing spaces and NUL characters removed. +inline auto to_string(const char* source, size_t size) -> std::string { size_t len = size; - while (len > 0 && (arr[len - 1] == ' ' || arr[len - 1] == '\0')) { + while (len > 0 && (source[len - 1] == ' ' || source[len - 1] == '\0')) { len--; } - return std::string {arr, len}; + return std::string {source, len}; } +/// @brief Per-message-type stream formatter family, used internally by +/// `print_message` to dispatch to the correct overload for whichever +/// alternative of `Message` it is given. +/// +/// One overload of `print_impl` exists for every message struct defined above. +/// Each overload writes a human-readable, field-by-field representation of its +/// message to `out`. +/// +/// @param out The output stream to write to. +/// @param msg The message to format. auto print_impl(std::ostream& out, const SystemEventMessage& msg) -> void; auto print_impl(std::ostream& out, const StockDirectoryMessage& msg) -> void; auto print_impl(std::ostream& out, const StockTradingActionMessage& msg) -> void; @@ -361,10 +534,19 @@ auto print_impl(std::ostream& out, const NOIIMessage& msg) -> void; auto print_impl(std::ostream& out, const RetailPriceImprovementIndicatorMessage& msg) -> void; auto print_impl(std::ostream& out, const DLCRMessage& msg) -> void; -// General print function that dispatches to the correct implementation +/// @brief Dispatches to the `print_impl` overload matching the active +/// alternative of `msg` and writes its formatted representation to `out`. +/// +/// @param out The output stream to write to. +/// @param msg The message to format; the concrete type formatted is whichever +/// alternative of the variant is currently held. auto print_message(std::ostream& out, const Message& msg) -> void; -// print an itch::Message +/// @brief Streams a formatted representation of `msg` to `out`. +/// +/// @param out The output stream to write to. +/// @param msg The message to format. +/// @return A reference to `out`, to allow chaining. auto operator<<(std::ostream& out, const Message& msg) -> std::ostream&; } // namespace itch diff --git a/include/itch/order_book.hpp b/include/itch/order_book.hpp index 824d335..0f6be61 100644 --- a/include/itch/order_book.hpp +++ b/include/itch/order_book.hpp @@ -1,12 +1,22 @@ #pragma once +/// @file +/// @brief An in-memory limit order book reconstructed from a stream of parsed +/// ITCH 5.0 messages. +/// +/// `LimitOrderBook` maintains the resting bid/ask price levels for a single +/// instrument, applying Add/Execute/Cancel/Delete/Replace order events as they +/// arrive so callers can query the current book depth or render it for +/// inspection. +/// +/// @author Bertin Balouki SIMYELI + #include #include -#include -#include #include #include #include +#include #include #include @@ -30,6 +40,13 @@ struct Order { PriceLevel* level; ///< Pointer to the price level containing this order. std::string stock; ///< The stock symbol for this order. + /// @brief Constructs an order with the given identity, side, quantity, and price. + /// + /// @param ref_num The exchange-assigned order reference number. + /// @param side 'B' for Buy, 'S' for Sell. + /// @param shrs The initial quantity of shares in the order. + /// @param prc The limit price of the order. + /// @param stk The stock symbol for this order. Order(uint64_t ref_num, char side, uint32_t shrs, uint32_t prc, const std::string& stk) : order_reference_number {ref_num}, buy_sell_indicator {side}, @@ -53,11 +70,11 @@ struct PriceLevel { /// @brief Appends an order to the end of the queue (Time priority). /// @param order Shared pointer to the order to add. - auto add_order(std::shared_ptr order) -> void; + auto add_order(const std::shared_ptr& order) -> void; /// @brief Removes a specific order from the queue. /// @param order_it Iterator pointing to the order to remove. - auto remove_order(OrderIt order_it) -> void; + auto remove_order(const OrderIt& order_it) -> void; }; /// @class LimitOrderBook @@ -72,6 +89,10 @@ struct PriceLevel { /// of price levels, and O(1) access to the best bid/ask. class LimitOrderBook { public: + /// @brief Constructs an order book scoped to a single stock symbol. + /// + /// @param stock_symbol The symbol of the instrument this book tracks; only + /// messages for this symbol affect the book state. LimitOrderBook(const std::string& stock_symbol) : m_stock_symbol(stock_symbol) {} /// @brief Dispatches and processes a generic ITCH message. /// @@ -95,9 +116,13 @@ class LimitOrderBook { using BidMap = std::map>; using AskMap = std::map>; + /// @brief The map of active Bids. + /// /// @return A reference to the map of active Bids, sorted descending by price. auto get_bids() const -> const BidMap& { return m_bids; } + /// @brief The map of active Asks. + /// /// @return A reference to the map of active Asks, sorted ascending by price. auto get_asks() const -> const AskMap& { return m_asks; } @@ -111,19 +136,63 @@ class LimitOrderBook { std::map m_orders; ///< Hash map for O(1) order lookup by Reference Number. // Message Handlers + /// @brief Handles an Add Order (no MPID) message: inserts a new resting + /// order into the book if it belongs to this instrument. + /// + /// @param msg The parsed Add Order message. auto handle_message(const AddOrderMessage& msg) -> void; + + /// @brief Handles an Add Order with MPID Attribution message: inserts a + /// new resting order into the book if it belongs to this instrument. + /// + /// @param msg The parsed Add Order (MPID Attribution) message. auto handle_message(const AddOrderMPIDAttributionMessage& msg) -> void; + + /// @brief Handles an Order Executed message: reduces or removes the + /// referenced order by its executed share quantity. + /// + /// @param msg The parsed Order Executed message. auto handle_message(const OrderExecutedMessage& msg) -> void; + + /// @brief Handles an Order Executed With Price message: reduces or removes + /// the referenced order by its executed share quantity. + /// + /// @param msg The parsed Order Executed With Price message. auto handle_message(const OrderExecutedWithPriceMessage& msg) -> void; + + /// @brief Handles an Order Cancel message: reduces or removes the + /// referenced order by its cancelled share quantity. + /// + /// @param msg The parsed Order Cancel message. auto handle_message(const OrderCancelMessage& msg) -> void; + + /// @brief Handles an Order Delete message: removes the referenced order + /// from the book entirely. + /// + /// @param msg The parsed Order Delete message. auto handle_message(const OrderDeleteMessage& msg) -> void; + + /// @brief Handles an Order Replace message: removes the original order and + /// inserts its replacement at the new reference number, quantity, + /// and price. + /// + /// @param msg The parsed Order Replace message. auto handle_message(const OrderReplaceMessage& msg) -> void; - // Generic handler/sink for message types that do not impact book state. + /// @brief Generic handler/sink for message types that do not impact book state. + /// + /// @tparam T The message type being ignored. + /// @param msg The message instance, ignored. template auto handle_message(const T& /* msg */) -> void { /* No-op for irrelevant messages */ } /// @brief Creates an Order object and inserts it into the appropriate PriceLevel. + /// + /// @param order_ref The exchange-assigned order reference number. + /// @param side 'B' for Buy, 'S' for Sell; selects the bid or ask side. + /// @param shares The initial quantity of shares in the order. + /// @param price The limit price of the order. + /// @param stock The stock symbol for this order. auto add_order( uint64_t order_ref, char side, uint32_t shares, uint32_t price, const std::string& stock ) -> void; diff --git a/include/itch/overlay.hpp b/include/itch/overlay.hpp new file mode 100644 index 0000000..6974468 --- /dev/null +++ b/include/itch/overlay.hpp @@ -0,0 +1,388 @@ +#pragma once + +/// @file +/// @brief Zero-copy typed views over raw ITCH message frames. +/// +/// Provides `MessageView` and a family of per-message-type `*View` subclasses +/// that read fields lazily, converting each field from network byte order on +/// access rather than eagerly decoding into a host-order struct. This contrasts +/// with the eager `Parser` in parser.hpp, which decodes every field up front. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include +#include +#include +#include + +#include "itch/detail/wire.hpp" +#include "itch/parser.hpp" + +namespace itch::overlay { + +namespace detail { + +/// @brief Reads an integral field of width sizeof(T) at `offset`, converting from +/// network (big-endian) order on access. +/// +/// @tparam FieldType The type of the field to read. +/// @param base Pointer to the start of the message frame. +/// @param offset Byte offset of the field within the frame. +/// @return The field value, converted from big-endian if it is a multi-byte +/// integral type. +template +[[nodiscard]] inline auto read_field(const std::byte* base, std::size_t offset) noexcept + -> FieldType { + FieldType value {}; + std::memcpy(&value, base + offset, sizeof(FieldType)); + if constexpr (std::is_integral_v && sizeof(FieldType) > 1) { + return utils::from_big_endian(value); + } else { + return value; + } +} + +/// @brief Reads the 48-bit ITCH timestamp at `offset` into a 64-bit value. +/// +/// @param base Pointer to the start of the message frame. +/// @param offset Byte offset of the timestamp field within the frame. +/// @return The timestamp, in nanoseconds past midnight. +[[nodiscard]] inline auto read_timestamp(const std::byte* base, std::size_t offset) noexcept + -> std::uint64_t { + const auto high = read_field(base, offset); + const auto low = read_field(base, offset + 2); + constexpr int LOWER_SHIFT = 32; + return (static_cast(high) << LOWER_SHIFT) | low; +} + +/// @brief Views an 8-byte fixed-width stock field as a string_view (untrimmed). +/// +/// @param base Pointer to the start of the message frame. +/// @param offset Byte offset of the stock field within the frame. +/// @return An untrimmed view of the 8-byte stock symbol field. +[[nodiscard]] inline auto read_stock(const std::byte* base, std::size_t offset) noexcept + -> std::string_view { + const void* raw = base + offset; + return std::string_view {static_cast(raw), 8}; +} + +// Common field offsets shared by every message after the 1-byte type. +constexpr std::size_t STOCK_LOCATE_OFFSET = 1; +constexpr std::size_t TRACKING_NUMBER_OFFSET = 3; +constexpr std::size_t TIMESTAMP_OFFSET = 5; + +} // namespace detail + +/// @brief A zero-copy typed view over one raw ITCH frame. +/// +/// In contrast to the eager `Parser`, which decodes every field of a message into +/// a host-order struct, an overlay view holds only a pointer to the frame in the +/// original buffer and converts each field from network byte order lazily, on the +/// access. For callers that touch only a few fields per message (for example a +/// filter that reads just the stock and price), this avoids decoding the fields +/// they never look at. The view is non-owning: it is valid only while the +/// underlying buffer lives. +class MessageView { + public: + /// @brief Constructs an empty view with no underlying frame. + constexpr MessageView() noexcept = default; + + /// @brief Constructs a view over a raw ITCH message frame. + /// + /// @param data Pointer to the start of the frame (the type byte). + /// @param size Size of the frame in bytes. + explicit MessageView(const std::byte* data, std::size_t size) noexcept + : m_data {data}, m_size {size} {} + + /// @brief The one-byte message type. + /// @return The one-byte message type. + [[nodiscard]] auto type() const noexcept -> char { + return static_cast(static_cast(m_data[0])); + } + + /// @brief The locate code identifying the security. + /// @return The locate code identifying the security. + [[nodiscard]] auto stock_locate() const noexcept -> std::uint16_t { + return detail::read_field(m_data, detail::STOCK_LOCATE_OFFSET); + } + + /// @brief The Nasdaq internal tracking number. + /// @return The Nasdaq internal tracking number. + [[nodiscard]] auto tracking_number() const noexcept -> std::uint16_t { + return detail::read_field(m_data, detail::TRACKING_NUMBER_OFFSET); + } + + /// @brief The message timestamp (nanoseconds past midnight). + /// @return The message timestamp (nanoseconds past midnight). + [[nodiscard]] auto timestamp() const noexcept -> std::uint64_t { + return detail::read_timestamp(m_data, detail::TIMESTAMP_OFFSET); + } + + /// @brief The raw frame size in bytes. + /// @return The raw frame size in bytes. + [[nodiscard]] auto size() const noexcept -> std::size_t { return m_size; } + + /// @brief Pointer to the raw frame (the type byte). + /// @return Pointer to the raw frame (the type byte). + [[nodiscard]] auto data() const noexcept -> const std::byte* { return m_data; } + + /// @brief Reads an arbitrary integral field at a byte offset, big-endian. + /// + /// @tparam FieldType The type of the field to read. + /// @param offset Byte offset of the field within the frame. + /// @return The field value, converted from big-endian if it is a multi-byte + /// integral type. + template + [[nodiscard]] auto read(std::size_t offset) const noexcept -> FieldType { + return detail::read_field(m_data, offset); + } + + protected: + const std::byte* m_data {nullptr}; + std::size_t m_size {0}; +}; + +/// @brief Lazy view of an Add Order (`A`) message. +class AddOrderView : public MessageView { + public: + using MessageView::MessageView; + + /// @brief Day-unique reference number for the order. + /// @return Day-unique reference number for the order. + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + + /// @brief 'B' buy, 'S' sell. + /// @return 'B' buy, 'S' sell. + [[nodiscard]] auto buy_sell_indicator() const noexcept -> char { return read(19); } + + /// @brief Displayed share quantity. + /// @return Displayed share quantity. + [[nodiscard]] auto shares() const noexcept -> std::uint32_t { return read(20); } + + /// @brief Stock symbol, right padded with spaces. + /// @return Stock symbol, right padded with spaces. + [[nodiscard]] auto stock() const noexcept -> std::string_view { + return detail::read_stock(m_data, 24); + } + + /// @brief Display price (4 implied decimals). + /// @return Display price (4 implied decimals). + [[nodiscard]] auto price() const noexcept -> std::uint32_t { return read(32); } +}; + +/// @brief Lazy view of an Order Executed (`E`) message. +class OrderExecutedView : public MessageView { + public: + using MessageView::MessageView; + + /// @brief Reference number of the executed order. + /// @return Reference number of the executed order. + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + + /// @brief Number of shares executed. + /// @return Number of shares executed. + [[nodiscard]] auto executed_shares() const noexcept -> std::uint32_t { + return read(19); + } + + /// @brief Day-unique match number for the execution. + /// @return Day-unique match number for the execution. + [[nodiscard]] auto match_number() const noexcept -> std::uint64_t { + return read(23); + } +}; + +/// @brief Lazy view of an Order Executed With Price (`C`) message. +class OrderExecutedWithPriceView : public MessageView { + public: + using MessageView::MessageView; + + /// @brief Reference number of the executed order. + /// @return Reference number of the executed order. + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + + /// @brief Number of shares executed. + /// @return Number of shares executed. + [[nodiscard]] auto executed_shares() const noexcept -> std::uint32_t { + return read(19); + } + + /// @brief Day-unique match number for the execution. + /// @return Day-unique match number for the execution. + [[nodiscard]] auto match_number() const noexcept -> std::uint64_t { + return read(23); + } + + /// @brief 'Y' if the trade is printable to the tape, else 'N'. + /// @return 'Y' if the trade is printable to the tape, else 'N'. + [[nodiscard]] auto printable() const noexcept -> char { return read(31); } + + /// @brief Price at which the order executed (4 decimals). + /// @return Price at which the order executed (4 decimals). + [[nodiscard]] auto execution_price() const noexcept -> std::uint32_t { + return read(32); + } +}; + +/// @brief Lazy view of an Order Cancel (`X`) message. +class OrderCancelView : public MessageView { + public: + using MessageView::MessageView; + + /// @brief Reference number of the cancelled order. + /// @return Reference number of the cancelled order. + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + + /// @brief Number of shares cancelled. + /// @return Number of shares cancelled. + [[nodiscard]] auto cancelled_shares() const noexcept -> std::uint32_t { + return read(19); + } +}; + +/// @brief Lazy view of an Order Delete (`D`) message. +class OrderDeleteView : public MessageView { + public: + using MessageView::MessageView; + + /// @brief Reference number of the deleted order. + /// @return Reference number of the deleted order. + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } +}; + +/// @brief Lazy view of an Order Replace (`U`) message. +class OrderReplaceView : public MessageView { + public: + using MessageView::MessageView; + + /// @brief Reference number being replaced. + /// @return Reference number being replaced. + [[nodiscard]] auto original_order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + + /// @brief New reference number for the order. + /// @return New reference number for the order. + [[nodiscard]] auto new_order_reference_number() const noexcept -> std::uint64_t { + return read(19); + } + + /// @brief New displayed share quantity. + /// @return New displayed share quantity. + [[nodiscard]] auto shares() const noexcept -> std::uint32_t { return read(27); } + + /// @brief New display price (4 implied decimals). + /// @return New display price (4 implied decimals). + [[nodiscard]] auto price() const noexcept -> std::uint32_t { return read(31); } +}; + +/// @brief Lazy view of a Trade (`P`, non-cross) message. +class NonCrossTradeView : public MessageView { + public: + using MessageView::MessageView; + + /// @brief Reference number of the non-displayed order. + /// @return Reference number of the non-displayed order. + [[nodiscard]] auto order_reference_number() const noexcept -> std::uint64_t { + return read(11); + } + + /// @brief 'B' buy, 'S' sell. + /// @return 'B' buy, 'S' sell. + [[nodiscard]] auto buy_sell_indicator() const noexcept -> char { return read(19); } + + /// @brief Number of shares traded. + /// @return Number of shares traded. + [[nodiscard]] auto shares() const noexcept -> std::uint32_t { return read(20); } + + /// @brief Stock symbol, right padded with spaces. + /// @return Stock symbol, right padded with spaces. + [[nodiscard]] auto stock() const noexcept -> std::string_view { + return detail::read_stock(m_data, 24); + } + + /// @brief Trade price (4 implied decimals). + /// @return Trade price (4 implied decimals). + [[nodiscard]] auto price() const noexcept -> std::uint32_t { return read(32); } + + /// @brief Day-unique match number for the trade. + /// @return Day-unique match number for the trade. + [[nodiscard]] auto match_number() const noexcept -> std::uint64_t { + return read(36); + } +}; + +/// @brief The signature for the overlay framing callback. +using ViewCallback = std::function; + +/// @brief Builds the per-type expected wire-size table at compile time, mirroring +/// the eager parser's so the overlay validates frame lengths identically. +/// +/// @return An array indexed by message-type byte, holding the expected wire size +/// for each known message type (zero for unknown types). +[[nodiscard]] consteval auto build_size_table() -> std::array { + std::array table {}; + auto add = [&table](char type) { + table[static_cast(type)] = + static_cast(itch::detail::WIRE_SIZE); + }; + itch::detail::for_each_message_type(add); + return table; +} + +inline constexpr auto SIZE_TABLE = build_size_table(); + +/// @brief Frames a buffer and invokes `callback` with a zero-copy `MessageView` +/// for each well-formed message. +/// +/// Framing and length validation match `Parser`: the 2-byte length prefix is +/// honoured, unknown type bytes and undersized frames are skipped. Unlike the +/// eager parser, no fields are decoded; the callback receives a view it can +/// inspect lazily. +/// +/// @param data The raw buffer containing one or more length-prefixed ITCH frames. +/// @param callback Invoked with a `MessageView` for each well-formed frame found. +/// @return The number of views delivered to `callback`. +inline auto for_each_message(std::span data, const ViewCallback& callback) + -> std::uint64_t { + std::uint64_t delivered = 0; + std::size_t offset = 0; + while (offset + sizeof(std::uint16_t) <= data.size()) { + std::uint16_t length {}; + std::memcpy(&length, data.data() + offset, sizeof(length)); + length = utils::from_big_endian(length); + offset += sizeof(std::uint16_t); + if (length == 0) { + continue; + } + if (offset + length > data.size()) { + break; + } + const std::byte* frame = data.data() + offset; + const auto message_type = static_cast(frame[0]); + offset += length; + + const std::uint16_t expected = SIZE_TABLE[message_type]; + if (expected == 0 || length < expected) { + continue; // Unknown type or undersized frame. + } + callback(MessageView {frame, length}); + ++delivered; + } + return delivered; +} + +} // namespace itch::overlay diff --git a/include/itch/parser.hpp b/include/itch/parser.hpp index 611988f..c8f33f0 100644 --- a/include/itch/parser.hpp +++ b/include/itch/parser.hpp @@ -1,9 +1,33 @@ #pragma once +/// @file +/// @brief Public parsing interface for NASDAQ TotalView-ITCH 5.0 feeds, plus +/// the low-level byte-unpacking utilities that back it. +/// +/// `Parser` frames and decodes a raw ITCH byte stream into the `Message` +/// variant defined in itch/messages.hpp, offering both throwing and +/// non-throwing (`std::expected`-based) entry points over buffers, spans, and +/// streams. The `itch::utils` helpers underneath handle endianness conversion +/// and fixed-width field extraction, and are also reused by the encoder. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include +#include +#include #include #include -#include +#include +#include #include +#include + +#ifdef __cpp_lib_expected +#include +#endif #include "itch/messages.hpp" @@ -15,6 +39,24 @@ namespace itch { /// @param const Message& A const reference to the fully parsed message object. using MessageCallback = std::function; +/// @brief Categories of recoverable problems encountered while framing a feed. +/// +/// These are surfaced through the non-throwing `try_parse` entry points and the +/// optional error callback, so callers can react without relying on exceptions +/// or on the library writing to a global stream. +enum class ParseError { + truncated, ///< The buffer ended in the middle of a frame. + unknown_type, ///< The message type byte does not correspond to a known message. + size_mismatch, ///< The declared length is shorter than the message type requires. +}; + +/// @brief The signature for the optional diagnostics callback. +/// +/// @param ParseError The category of problem that occurred. +/// @param char The offending message type byte (`'\0'` when not applicable, +/// e.g. for a truncated header). +using ErrorCallback = std::function; + /// @brief A high-performance parser for the NASDAQ TotalView-ITCH 5.0 protocol. /// /// This class is designed to parse a raw binary feed of ITCH 5.0 messages, @@ -37,11 +79,9 @@ class Parser { public: /// @brief Constructs a Parser instance. /// - /// The constructor pre-populates an internal dispatch table (a map of - /// handlers) by registering a specific parsing function for each known ITCH - /// message type. This setup makes the parsing process a fast lookup - /// operation at runtime. - Parser(); + /// Dispatch is driven by a compile-time table keyed on the message type + /// byte, so no per-instance setup is required. + Parser() = default; /// @brief Parses messages from a memory buffer and invokes a callback for /// each. @@ -129,58 +169,186 @@ class Parser { /// @throw std::runtime_error on stream reading errors. auto parse(std::istream& data, const std::vector& messages) -> std::vector; + /// @brief Parses messages from a byte span and invokes a callback for each. + /// + /// The `std::span` overloads are the preferred modern interface: the span + /// carries its own size, which prevents the pointer/length desynchronization + /// that the raw `(const char*, size_t)` overloads are prone to. Those raw + /// overloads are retained as thin shims for C interop. + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @param callback A function to be called for each successfully parsed + /// message. + /// @throw std::runtime_error if the buffer ends in the middle of a message. + auto parse(std::span data, const MessageCallback& callback) -> void; + + /// @brief Parses all messages from a byte span and returns them in a vector. + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @return A std::vector containing all parsed messages. + /// @throw std::runtime_error if the buffer ends in the middle of a message. + auto parse(std::span data) -> std::vector; + + /// @brief Parses messages from a byte span, keeping only the requested types. + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @param messages A vector of message type characters to keep (e.g., {'A', 'P'}). + /// @return A std::vector containing only the filtered messages. + /// @throw std::runtime_error if the buffer ends in the middle of a message. + auto parse(std::span data, const std::vector& messages) + -> std::vector; + +#ifdef __cpp_lib_expected + /// @brief Non-throwing parse: invokes a callback per message, reporting + /// truncation through the return value instead of an exception. + /// + /// This is the latency-friendly entry point for callers that prefer not to + /// pay for exceptions on the hot path. Unknown message types and oversized + /// or undersized frames are routed to the diagnostics policy (counters plus + /// the optional error callback) and skipped; only an unrecoverable + /// truncation yields an `unexpected` result. + /// + /// Available when the standard library provides `std::expected` (C++23). + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @param callback A function to be called for each successfully parsed + /// message. + /// @return Nothing on success, or a `ParseError` describing the failure. + [[nodiscard]] auto try_parse(std::span data, const MessageCallback& callback) + -> std::expected; + + /// @brief Non-throwing parse that collects all messages into a vector. + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @return All parsed messages on success, or a `ParseError` describing the failure. + [[nodiscard]] auto try_parse(std::span data + ) -> std::expected, ParseError>; +#endif + + /// @brief Registers a callback invoked for each recoverable framing problem. + /// + /// Passing an empty function clears any previously installed callback. The + /// default behavior, with no callback installed, is to silently skip and + /// count the offending frame. + /// + /// @param callback The diagnostics callback to install, or an empty function + /// to clear any previously installed callback. + auto set_error_callback(ErrorCallback callback) -> void; + + /// @brief The number of frames skipped because their type byte was unknown. + /// + /// @return The running count of frames skipped for an unrecognized type byte. + [[nodiscard]] auto unknown_message_count() const noexcept -> std::uint64_t { + return m_unknown_message_count; + } + + /// @brief The number of frames skipped because their declared length was too + /// small for the message type. + /// + /// @return The running count of frames skipped for a declared length too + /// small for their message type. + [[nodiscard]] auto malformed_message_count() const noexcept -> std::uint64_t { + return m_malformed_message_count; + } + + /// @brief Resets the accumulating diagnostics counters to zero. + auto reset_diagnostics() noexcept -> void { + m_unknown_message_count = 0; + m_malformed_message_count = 0; + } + private: - using Handler = std::function; - std::map m_handlers; - - /// @brief A template function to register a handler for a specific - /// message type. - /// @tparam T The C++ struct type representing the message (e.g., - /// AddOrderMessage). - /// @param type The character identifier for this message type (e.g., - /// 'A'). - template - auto register_handler(char type) -> void; + /// @brief The shared, non-throwing framing loop backing every parse overload. + /// + /// @param data A pointer to the start of the memory buffer containing ITCH data. + /// @param size The total size of the buffer in bytes. + /// @param callback A function to be called for each successfully parsed message. + /// @return `std::nullopt` on success, or the `ParseError` that aborted the + /// loop (only an unrecoverable truncation aborts it). + auto parse_impl(const char* data, std::size_t size, const MessageCallback& callback) + -> std::optional; + + /// @brief Records a recoverable framing problem and notifies the callback. + /// + /// @param error The category of problem that occurred. + /// @param message_type The offending message type byte (`'\0'` when not + /// applicable, e.g. for a truncated header). + auto report_error(ParseError error, char message_type) -> void; + + ErrorCallback m_error_callback {}; + std::uint64_t m_unknown_message_count {0}; + std::uint64_t m_malformed_message_count {0}; }; namespace utils { -// Generic byte-swapping function for any integral type. -template -T swap_bytes(T value) { - static_assert(std::is_integral_v, "swap_bytes can only be used with integral types"); - union { - T val; - uint8_t bytes[sizeof(T)]; - } src, dst; - src.val = value; - for (size_t i = 0; i < sizeof(T); ++i) { - dst.bytes[i] = src.bytes[sizeof(T) - 1 - i]; +/// @brief Reverses the byte order of an integral value. +/// +/// Prefers `std::byteswap` (C++23) and otherwise falls back to a well-defined +/// `std::bit_cast` based reversal. This deliberately avoids reading an inactive +/// union member, which is undefined behavior in C++ and a hazard on the parser +/// hot path. +/// +/// @tparam IntType The integral type to byte-swap. +/// @param value The value whose byte order should be reversed. +/// @return `value` with its byte order reversed. +template +[[nodiscard]] constexpr auto swap_bytes(IntType value) noexcept -> IntType { +#ifdef __cpp_lib_byteswap + return std::byteswap(value); +#else + if constexpr (sizeof(IntType) == 1) { + return value; + } else { + auto bytes = std::bit_cast>(value); + std::ranges::reverse(bytes); + return std::bit_cast(bytes); } - return dst.val; +#endif } -// Check the system's endianness at compile time (or runtime as a fallback). -// This determines if we need to swap bytes at all. -inline bool is_little_endian(); - -// Converts a value from big-endian (network order) to the host's native byte -// order. On little-endian systems (like x86/x64), it swaps the bytes. On -// big-endian systems, it does nothing. -template -T from_big_endian(T value); +/// @brief Converts a big-endian (network order) value to the host byte order. +/// +/// On little-endian hosts this swaps the bytes; on big-endian hosts it is a +/// no-op. The choice is made at compile time via `std::endian`. +/// +/// @tparam IntType The integral type to convert. +/// @param value The big-endian value to convert. +/// @return `value` converted to host byte order. +template +[[nodiscard]] constexpr auto from_big_endian(IntType value) noexcept -> IntType { + if constexpr (std::endian::native == std::endian::little) { + return swap_bytes(value); + } else { + return value; + } +} -// Unpacks a value of type T from the buffer at the given offset, updating -// the offset accordingly. Handles endianness conversion for integral types. -template -T unpack(const char* buffer, size_t& offset); +/// @brief Unpacks a value of type T from the buffer, advancing the offset. +/// Handles endianness conversion for multi-byte integral types. +/// +/// @tparam ValueType The type of value to unpack. +/// @param buffer The buffer to read from. +/// @param offset The byte offset to read from, advanced past the unpacked value. +/// @return The unpacked value, converted to host byte order when applicable. +template +auto unpack(const char* buffer, std::size_t& offset) -> ValueType; -// Specialization for char arrays (strings) -inline void unpack_string(const char* buffer, size_t& offset, char* dest, size_t size); +/// @brief Copies a fixed-width character field out of the buffer. +/// +/// @param buffer The buffer to read from. +/// @param offset The byte offset to read from, advanced past the copied field. +/// @param dest The destination buffer to copy the field into. +/// @param size The number of characters to copy. +inline auto unpack_string(const char* buffer, std::size_t& offset, char* dest, std::size_t size) + -> void; -// ITCH timestamps are 48-bit big-endian integers. -// 2 bytes for high part, 4 bytes for low part. -inline uint64_t unpack_timestamp(const char* buffer, size_t& offset); +/// @brief Unpacks a 48-bit big-endian ITCH timestamp (2-byte high, 4-byte low). +/// +/// @param buffer The buffer to read from. +/// @param offset The byte offset to read from, advanced past the timestamp. +/// @return The timestamp as nanoseconds past midnight. +inline auto unpack_timestamp(const char* buffer, std::size_t& offset) -> std::uint64_t; } // namespace utils } // namespace itch diff --git a/include/itch/price.hpp b/include/itch/price.hpp new file mode 100644 index 0000000..37f9c18 --- /dev/null +++ b/include/itch/price.hpp @@ -0,0 +1,137 @@ +#pragma once + +/// @file +/// @brief Strongly typed, fixed-point price representation for ITCH price fields. +/// +/// Wraps the raw on-wire integer price in a type that carries its own decimal +/// scale, so standard 4-decimal prices and MWCB 8-decimal decline-level prices +/// cannot be mixed, compared, or formatted with the wrong divisor by accident. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include + +namespace itch { + +/// @brief A strongly typed fixed-point price carrying its own decimal scale. +/// +/// ITCH prices are wire integers with an implied number of decimal places. +/// Standard price fields imply 4 decimals (divisor 10,000); MWCB decline-level +/// prices imply 8 decimals (divisor 100,000,000). Exposing both as a bare +/// `uint32_t`/`uint64_t` makes it trivially easy to divide by the wrong scale +/// and silently produce a wrong price. +/// +/// `BasicPrice` encodes the scale in the type. The two scales are therefore +/// distinct types (`StandardPrice` vs `MwcbPrice`) that cannot be mixed, +/// compared, or assigned to one another by accident: the 4-vs-8 decimal mistake +/// becomes a compile error rather than a runtime bug. +/// +/// @tparam RawType The unsigned integer type of the on-wire value. +/// @tparam Decimals The number of implied decimal places. +template +class BasicPrice { + static_assert(std::is_unsigned_v, "Price raw storage must be an unsigned integer"); + + public: + using raw_type = RawType; + + /// @brief The number of implied decimal places for this price scale. + static constexpr unsigned int decimals = Decimals; + + /// @brief Constructs a zero-valued price at this scale. + constexpr BasicPrice() noexcept = default; + + /// @brief Wraps a raw on-wire price value at this scale. + /// + /// @param raw_value The raw integer price, exactly as it appears on the wire. + explicit constexpr BasicPrice(raw_type raw_value) noexcept : m_raw {raw_value} {} + + /// @brief The underlying raw integer value, exactly as it appears on the wire. + /// + /// @return The raw integer price at this scale. + [[nodiscard]] constexpr auto raw() const noexcept -> raw_type { return m_raw; } + + /// @brief The scale's divisor, i.e. 10^Decimals. + /// + /// @return The divisor used to convert the raw integer value into a + /// floating-point or decimal-string representation. + [[nodiscard]] static constexpr auto divisor() noexcept -> raw_type { + raw_type result {1}; + for (unsigned int exponent {0}; exponent < Decimals; ++exponent) { + result *= 10; + } + return result; + } + + /// @brief The price as a floating-point value (raw / 10^Decimals). + /// + /// @return The price converted to a `double`. + [[nodiscard]] constexpr auto to_double() const noexcept -> double { + return static_cast(m_raw) / static_cast(divisor()); + } + + /// @brief The price as a fixed-precision decimal string. + /// + /// @return The price formatted with exactly `Decimals` fractional digits. + [[nodiscard]] auto to_string() const -> std::string { + return std::format("{:.{}f}", to_double(), Decimals); + } + + /// @brief Compares two prices of the same scale for equality by raw value. + /// + /// @param lhs The left-hand price. + /// @param rhs The right-hand price. + /// @return `true` if both prices have the same raw value. + [[nodiscard]] friend constexpr auto operator==(BasicPrice, BasicPrice) noexcept -> bool = + default; + /// @brief Orders two prices of the same scale by raw value. + /// + /// @param lhs The left-hand price. + /// @param rhs The right-hand price. + /// @return The three-way comparison result between `lhs` and `rhs`. + [[nodiscard]] friend constexpr auto operator<=>(BasicPrice, BasicPrice) noexcept = default; + + private: + raw_type m_raw {0}; +}; + +/// @brief Price scale for every ITCH price field except MWCB decline levels. +using StandardPrice = BasicPrice; + +/// @brief Price scale for MWCB decline-level prices (8 implied decimals). +using MwcbPrice = BasicPrice; + +/// @brief Wraps a raw 4-decimal price value in the typed StandardPrice. +/// +/// @param raw_value The raw on-wire 4-decimal price. +/// @return The value wrapped as a `StandardPrice`. +[[nodiscard]] constexpr auto make_price(std::uint32_t raw_value) noexcept -> StandardPrice { + return StandardPrice {raw_value}; +} + +/// @brief Wraps a raw 8-decimal MWCB price value in the typed MwcbPrice. +/// +/// @param raw_value The raw on-wire 8-decimal MWCB decline-level price. +/// @return The value wrapped as an `MwcbPrice`. +[[nodiscard]] constexpr auto make_mwcb_price(std::uint64_t raw_value) noexcept -> MwcbPrice { + return MwcbPrice {raw_value}; +} + +} // namespace itch + +/// @brief Formats a price with its scale's number of decimals, e.g. "150.0000". +template +struct std::formatter> : std::formatter { + /// @brief Formats `price` as a fixed-precision decimal string. + /// + /// @param price The price to format. + /// @param ctx The format context to write the formatted output into. + /// @return An output iterator positioned after the formatted text, per the + /// `std::formatter` contract. + auto format(itch::BasicPrice price, std::format_context& ctx) const { + return std::formatter::format(price.to_double(), ctx); + } +}; diff --git a/include/itch/replay.hpp b/include/itch/replay.hpp new file mode 100644 index 0000000..b093cef --- /dev/null +++ b/include/itch/replay.hpp @@ -0,0 +1,63 @@ +#pragma once + +/// @file +/// @brief A timestamp-paced replay engine for previously captured ITCH streams. +/// +/// Wraps the `Parser` so a consumer can play back a buffer at (a multiple of) +/// its original wall-clock cadence rather than as fast as the CPU can parse +/// it, which is useful for realistic backtesting and system simulation. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include + +#include "itch/parser.hpp" + +namespace itch { + +/// @brief Replays a parsed ITCH stream paced by message timestamps. +/// +/// For realistic backtesting and system simulation, a consumer often needs the +/// feed delivered at its original wall-clock cadence rather than as fast as the +/// CPU can parse it. The replay engine parses a buffer and invokes the callback +/// for each message, sleeping between messages so the inter-message gaps match the +/// differences in their nanoseconds-past-midnight timestamps, optionally scaled by +/// a speed factor. +class ReplayEngine { + public: + /// @brief Constructs an engine with a speed multiplier. + /// + /// @param speed_multiplier Wall-clock speed relative to the original feed: 1.0 + /// replays in real time, 2.0 twice as fast, 0.5 half speed. A value + /// less than or equal to 0 replays with no pacing (as fast as possible). + explicit ReplayEngine(double speed_multiplier = 1.0) noexcept + : m_speed_multiplier {speed_multiplier} {} + + /// @brief Sets the speed multiplier (see the constructor). + /// + /// @param speed_multiplier Wall-clock speed relative to the original feed + /// (see the constructor for the exact semantics). + auto set_speed(double speed_multiplier) noexcept -> void { + m_speed_multiplier = speed_multiplier; + } + + /// @brief The current speed multiplier. + /// + /// @return The currently configured speed multiplier. + [[nodiscard]] auto speed() const noexcept -> double { return m_speed_multiplier; } + + /// @brief Parses `data` and invokes `callback` for each message, paced by the + /// message timestamps. + /// + /// @param data A view over the contiguous buffer containing ITCH data. + /// @param callback A function to be called for each successfully parsed message. + /// @return The number of messages replayed. + auto replay(std::span data, const MessageCallback& callback) const + -> std::uint64_t; + + private: + double m_speed_multiplier {1.0}; +}; + +} // namespace itch diff --git a/include/itch/tape.hpp b/include/itch/tape.hpp new file mode 100644 index 0000000..b294861 --- /dev/null +++ b/include/itch/tape.hpp @@ -0,0 +1,44 @@ +#pragma once + +/// @file +/// @brief A normalized trade record and callback type for consuming the ITCH +/// trade tape. +/// +/// Execution-related ITCH message types are heterogeneous in shape; this +/// header defines the single, flattened `Trade` record that callers consume +/// instead of switching over the raw message variant themselves. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include + +#include "itch/price.hpp" + +namespace itch { + +/// @brief A single execution extracted from the feed (the "trade tape"). +/// +/// Trades arrive in several ITCH message types: executions of displayed orders +/// (`E`), executions at a different price (`C`), trades of non-displayable orders +/// (`P`), and cross trades (`Q`). They are normalized here into one record. The +/// `printable` flag carries the spec's displayable/non-displayable distinction so +/// consumers can correctly separate lit prints from hidden ones. +struct Trade { + std::uint64_t timestamp {0}; ///< Nanoseconds past midnight. + std::uint16_t stock_locate {0}; ///< Locate code identifying the security. + std::string symbol; ///< Stock symbol (may be empty if unknown). + StandardPrice price {}; ///< Execution price. + std::uint64_t shares {0}; ///< Executed share quantity. + std::uint64_t match_number {0}; ///< Exchange match number. + char side {'\0'}; ///< Resting order side ('B'/'S'), or '\0'. + bool printable {true}; ///< Whether the print is displayable. + bool is_cross {false}; ///< Whether this is a cross (auction) trade. + char cross_type {'\0'}; ///< Cross type for `Q` trades, else '\0'. +}; + +/// @brief Callback invoked for each extracted trade. +using TradeCallback = std::function; + +} // namespace itch diff --git a/include/itch/time.hpp b/include/itch/time.hpp new file mode 100644 index 0000000..fdc2e0c --- /dev/null +++ b/include/itch/time.hpp @@ -0,0 +1,60 @@ +#pragma once + +/// @file +/// @brief Helpers for converting raw ITCH nanoseconds-past-midnight timestamps +/// into absolute time points and human-readable strings. +/// +/// ITCH timestamps carry no date or time-zone of their own, so every +/// conversion here takes the session date as an explicit parameter and +/// performs no time-zone adjustment. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include + +namespace itch { + +/// @brief A system-clock time point at nanosecond resolution. +using ItchTimePoint = std::chrono::sys_time; + +/// @brief Combines a session date with an ITCH timestamp into a time point. +/// +/// ITCH timestamps are raw nanoseconds elapsed since midnight, with no date or +/// time-zone attached. This helper anchors that offset to the midnight of a +/// caller-supplied session date. No time-zone conversion is performed: the +/// result is expressed in the same frame as the date you pass in (pass a UTC +/// date for a UTC result, an Eastern-time date for an Eastern result). +/// +/// @param session_date The calendar date whose midnight anchors the timestamp. +/// @param nanos_past_midnight The raw ITCH timestamp (nanoseconds past midnight). +/// @return The absolute time point for the event. +[[nodiscard]] constexpr auto to_time_point( + std::chrono::year_month_day session_date, std::uint64_t nanos_past_midnight +) noexcept -> ItchTimePoint { + return std::chrono::sys_days {session_date} + std::chrono::nanoseconds {nanos_past_midnight}; +} + +/// @brief Formats a raw ITCH timestamp as a time-of-day string. +/// +/// @param nanos_past_midnight The raw ITCH timestamp (nanoseconds past midnight). +/// @return A string of the form "HH:MM:SS.nnnnnnnnn". +[[nodiscard]] inline auto format_timestamp(std::uint64_t nanos_past_midnight) -> std::string { + const std::chrono::hh_mm_ss time_of_day {std::chrono::nanoseconds {nanos_past_midnight}}; + return std::format("{:%T}", time_of_day); +} + +/// @brief Formats a session date plus an ITCH timestamp as a full date-time. +/// +/// @param session_date The calendar date whose midnight anchors the timestamp. +/// @param nanos_past_midnight The raw ITCH timestamp (nanoseconds past midnight). +/// @return A string of the form "YYYY-MM-DD HH:MM:SS.nnnnnnnnn". +[[nodiscard]] inline auto format_time_point( + std::chrono::year_month_day session_date, std::uint64_t nanos_past_midnight +) -> std::string { + return std::format("{:%F %T}", to_time_point(session_date, nanos_past_midnight)); +} + +} // namespace itch diff --git a/include/itch/transport/moldudp64.hpp b/include/itch/transport/moldudp64.hpp new file mode 100644 index 0000000..b979e79 --- /dev/null +++ b/include/itch/transport/moldudp64.hpp @@ -0,0 +1,115 @@ +#pragma once + +/// @file +/// @brief Decoder for NASDAQ's MoldUDP64 UDP multicast market-data framing. +/// +/// This header declares the `MoldUdp64Header` wire structure and the +/// `MoldUdp64Decoder` that strips it from each datagram and feeds the +/// contained, length-prefixed ITCH message blocks through the shared +/// `Parser`. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include +#include +#include + +#include "itch/parser.hpp" +#include "itch/transport/sequencing.hpp" + +namespace itch::transport { + +/// @brief The fixed 20-byte header that prefixes every MoldUDP64 downstream +/// packet. +/// +/// MoldUDP64 is the lightweight UDP multicast framing NASDAQ uses to disseminate +/// live market data. Each datagram carries a session identifier, the sequence +/// number of the first message it contains, and a count of the message blocks +/// that follow. The message blocks themselves are length-prefixed exactly like a +/// raw ITCH stream, so once the header is stripped the remainder feeds straight +/// into the ordinary `Parser` framing loop. +struct MoldUdp64Header { + std::array session {}; ///< Session id, space padded. + std::uint64_t sequence_number {0}; ///< Sequence of the first message block. + std::uint16_t message_count {0}; ///< Number of message blocks in the packet. + + /// @brief Sentinel `message_count` indicating an end-of-session packet. + static constexpr std::uint16_t END_OF_SESSION = 0xFFFF; + + /// @brief The session id as a view with trailing padding removed. + /// @return A view of the session id with trailing spaces/NULs stripped. + [[nodiscard]] auto session_view() const -> std::string_view { + std::size_t length = session.size(); + while (length > 0 && (session[length - 1] == ' ' || session[length - 1] == '\0')) { + --length; + } + return std::string_view {session.data(), length}; + } + + /// @brief A heartbeat carries no message blocks (`message_count == 0`). + /// @return True if the packet is a heartbeat, false otherwise. + [[nodiscard]] auto is_heartbeat() const noexcept -> bool { return message_count == 0; } + + /// @brief End-of-session is signalled by the sentinel `message_count`. + /// @return True if the packet signals end of session, false otherwise. + [[nodiscard]] auto is_end_of_session() const noexcept -> bool { + return message_count == END_OF_SESSION; + } +}; + +/// @brief Decodes MoldUDP64 datagrams and forwards the contained ITCH messages. +/// +/// One instance decodes a stream of datagrams from one or more sessions: call +/// `decode_packet` once per UDP payload. Each well-formed data packet's message +/// blocks are handed to an internal `Parser`, which invokes the supplied +/// `MessageCallback` for every decoded ITCH message. Per-packet sequence numbers +/// are fed to an embedded `SequenceTracker` so gaps in the multicast stream are +/// surfaced to the caller. +class MoldUdp64Decoder { + public: + /// @brief The on-wire size of the MoldUDP64 packet header, in bytes. + static constexpr std::size_t HEADER_SIZE = 20; + + /// @brief Constructs a decoder that calls `callback` for each ITCH message. + /// @param callback Invoked with each ITCH message decoded from a data + /// packet's message blocks. + explicit MoldUdp64Decoder(MessageCallback callback); + + /// @brief Decodes a single MoldUDP64 datagram. + /// + /// @param packet The full UDP payload (header plus message blocks). + /// @return The parsed header, or `std::nullopt` if the datagram is too short + /// to contain a valid header. + auto decode_packet(std::span packet) -> std::optional; + + /// @brief The embedded sequence tracker (install gap callbacks here). + /// @return Reference to the embedded `SequenceTracker`. + [[nodiscard]] auto tracker() noexcept -> SequenceTracker& { return m_tracker; } + /// @brief The embedded sequence tracker (install gap callbacks here). + /// @return Const reference to the embedded `SequenceTracker`. + [[nodiscard]] auto tracker() const noexcept -> const SequenceTracker& { return m_tracker; } + + /// @brief Total number of datagrams passed to `decode_packet`. + /// @return The count of datagrams decoded so far. + [[nodiscard]] auto packets_decoded() const noexcept -> std::uint64_t { + return m_packets_decoded; + } + + /// @brief Total number of ITCH messages decoded from all datagrams. + /// @return The total count of decoded ITCH messages. + [[nodiscard]] auto messages_decoded() const noexcept -> std::uint64_t { + return m_messages_decoded; + } + + private: + Parser m_parser {}; + MessageCallback m_callback; + SequenceTracker m_tracker {}; + std::uint64_t m_packets_decoded {0}; + std::uint64_t m_messages_decoded {0}; +}; + +} // namespace itch::transport diff --git a/include/itch/transport/pcap.hpp b/include/itch/transport/pcap.hpp new file mode 100644 index 0000000..b136b7d --- /dev/null +++ b/include/itch/transport/pcap.hpp @@ -0,0 +1,129 @@ +#pragma once + +/// @file +/// @brief In-house reader for classic pcap and pcapng capture files carrying +/// MoldUDP64/ITCH traffic. +/// +/// This header declares `PcapReader`, which walks the Ethernet/IPv4/IPv6/UDP +/// layers of each captured frame, extracts matching UDP payloads, and feeds +/// them through an embedded `MoldUdp64Decoder` to produce decoded ITCH +/// messages, without depending on libpcap. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include +#include +#include + +#include "itch/parser.hpp" +#include "itch/transport/moldudp64.hpp" + +namespace itch::transport { + +/// @brief Replays ITCH market data straight from a captured network trace. +/// +/// Firms most often archive and share feeds as packet captures, so being able to +/// consume a `.pcap`/`.pcapng` file directly removes the need to pre-strip the +/// transport framing. The reader is implemented entirely in-house (no libpcap +/// dependency): it understands the classic pcap and the pcapng container formats, +/// walks the Ethernet / IPv4 / IPv6 / UDP layers of each captured frame, extracts +/// the UDP payload, and feeds it through an embedded `MoldUdp64Decoder`, which in +/// turn yields the decoded ITCH messages to the caller's callback. +/// +/// @note Only offline capture files are supported. Live capture would require a +/// platform packet-capture library and is tracked as future work behind a +/// separate build option. +class PcapReader { + public: + /// @brief Constructs a reader that forwards each decoded ITCH message to + /// `callback`. + /// + /// @param callback Invoked with each ITCH message decoded from the + /// capture. + explicit PcapReader(MessageCallback callback); + + /// @brief Decodes an in-memory capture buffer. + /// + /// @param capture The full contents of a `.pcap` or `.pcapng` file. + /// @return `true` if the buffer was a recognized capture format, `false` + /// otherwise (in which case nothing was decoded). + auto read(std::span capture) -> bool; + + /// @brief Reads and decodes a capture file from disk. + /// + /// @param path Filesystem path to the `.pcap` or `.pcapng` file to read. + /// @return `true` on a recognized, readable file; `false` if the file cannot + /// be opened or is not a capture. + auto read_file(const std::string& path) -> bool; + + /// @brief Restricts decoding to UDP datagrams sent to this destination port. + /// When unset, datagrams on any port are decoded. + /// + /// @param port The destination UDP port to accept. + auto set_udp_port_filter(std::uint16_t port) -> void { m_port_filter = port; } + + /// @brief Clears any destination-port filter previously installed. + auto clear_udp_port_filter() noexcept -> void { m_port_filter = std::nullopt; } + + /// @brief The embedded MoldUDP64 decoder (sequence tracking lives here). + /// @return Reference to the embedded `MoldUdp64Decoder`. + [[nodiscard]] auto mold_decoder() noexcept -> MoldUdp64Decoder& { return m_mold; } + /// @brief The embedded MoldUDP64 decoder (sequence tracking lives here). + /// @return Const reference to the embedded `MoldUdp64Decoder`. + [[nodiscard]] auto mold_decoder() const noexcept -> const MoldUdp64Decoder& { return m_mold; } + + /// @brief Number of UDP datagrams extracted and handed to the MoldUDP64 + /// decoder. + /// @return The count of UDP datagrams processed so far. + [[nodiscard]] auto udp_datagrams() const noexcept -> std::uint64_t { return m_udp_datagrams; } + + /// @brief Total ITCH messages decoded across the whole capture. + /// @return The total count of decoded ITCH messages. + [[nodiscard]] auto messages_decoded() const noexcept -> std::uint64_t { + return m_mold.messages_decoded(); + } + + private: + /// @brief Parses the buffer as a classic pcap file and decodes its frames. + /// + /// @param capture The full contents of the candidate capture file. + /// @param swapped Whether the file's magic number indicated byte-swapped + /// (opposite-endian) header fields. + /// @return `true` if the buffer was a valid classic pcap file, `false` if + /// it is not that format or is structurally invalid. + auto read_classic_pcap(std::span capture, bool swapped) -> bool; + + /// @brief Parses the buffer as a pcapng file and decodes its frames. + /// + /// @param capture The full contents of the candidate capture file. + /// @return `true` if the buffer was a valid pcapng file, `false` if it is + /// not that format or is structurally invalid. + auto read_pcapng(std::span capture) -> bool; + + /// @brief Walks a single captured frame of the given link type and, if it + /// carries a matching UDP datagram, feeds the payload to the + /// MoldUDP64 decoder. + /// + /// @param frame The raw captured frame bytes (link-layer through payload). + /// @param link_type The capture's link-layer type (for example Ethernet), + /// identifying how to interpret `frame`. + auto handle_frame(std::span frame, std::uint32_t link_type) -> void; + + /// @brief Locates the UDP payload within a network-layer (IPv4/IPv6) span, + /// applying the configured destination-port filter. + /// + /// @param network The network-layer bytes (IP header onward). + /// @return The UDP payload if `network` carries a matching UDP datagram, + /// `std::nullopt` otherwise. + [[nodiscard]] auto extract_udp_payload(std::span network + ) const -> std::optional>; + + MoldUdp64Decoder m_mold; + std::optional m_port_filter {}; + std::uint64_t m_udp_datagrams {0}; +}; + +} // namespace itch::transport diff --git a/include/itch/transport/sequencing.hpp b/include/itch/transport/sequencing.hpp new file mode 100644 index 0000000..b3001a8 --- /dev/null +++ b/include/itch/transport/sequencing.hpp @@ -0,0 +1,168 @@ +#pragma once + +/// @file +/// @brief Per-session sequence-gap detection shared by the MoldUDP64 and +/// SoupBinTCP transport decoders. +/// +/// This header declares the `RetransmitRequester` hook a caller implements to +/// drive gap recovery, and the `SequenceTracker` that watches per-packet +/// sequence numbers and reports gaps through a callback and/or the requester. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include +#include +#include +#include + +namespace itch::transport { + +/// @brief Abstract hook a caller implements to drive recovery against a replay +/// or retransmission service when a sequence gap is detected. +/// +/// The library never performs network I/O itself: when `SequenceTracker` sees a +/// gap it calls `request_retransmit`, and the caller is responsible for issuing +/// the actual re-request (for example a SoupBinTCP/MoldUDP64 rewind request) and +/// feeding the recovered messages back through the decoder. +class RetransmitRequester { + public: + /// @brief Constructs a requester with no state. + RetransmitRequester() = default; + /// @brief Copy-constructs a requester. + /// @param other The requester to copy. + RetransmitRequester(const RetransmitRequester&) = default; + /// @brief Move-constructs a requester. + /// @param other The requester to move from. + RetransmitRequester(RetransmitRequester&&) noexcept = default; + /// @brief Copy-assigns a requester. + /// @param other The requester to copy. + /// @return Reference to this requester. + auto operator=(const RetransmitRequester&) -> RetransmitRequester& = default; + /// @brief Move-assigns a requester. + /// @param other The requester to move from. + /// @return Reference to this requester. + auto operator=(RetransmitRequester&&) noexcept -> RetransmitRequester& = default; + /// @brief Destroys the requester. + virtual ~RetransmitRequester() = default; + + /// @brief Requests retransmission of `count` messages starting at sequence + /// `start_sequence` for the given session. + /// + /// @param session The session identifier the gap was observed on. + /// @param start_sequence The first missing sequence number to retransmit. + /// @param count The number of missing messages to retransmit. + virtual auto request_retransmit( + std::string_view session, std::uint64_t start_sequence, std::uint64_t count + ) -> void = 0; +}; + +/// @brief Tracks per-session sequence numbers across a transport layer and +/// surfaces gaps. +/// +/// Both the MoldUDP64 and SoupBinTCP decoders feed their per-packet sequence +/// information here. The tracker maintains, per session, the next sequence number +/// it expects to see. When an observed packet starts ahead of that expectation it +/// reports a gap (through the gap callback and the optional `RetransmitRequester`) +/// and resynchronizes. Duplicate or already-seen sequences are ignored so a +/// replayed recovery stream does not double-count. +class SequenceTracker { + public: + /// @brief Invoked once per detected gap with the session, the sequence the + /// tracker expected, and the (higher) sequence actually received. + using GapCallback = std::function< + void(std::string_view session, std::uint64_t expected, std::uint64_t received)>; + + /// @brief Records that a packet beginning at `first_sequence` carried `count` + /// sequenced messages for `session`. + /// + /// @param session The session identifier the packet belongs to. + /// @param first_sequence The sequence number of the first message in the + /// packet. + /// @param count The number of sequenced messages carried by the packet. + /// @return The number of missing messages detected (0 when in order). + auto observe(std::string_view session, std::uint64_t first_sequence, std::uint64_t count) + -> std::uint64_t { + const std::string key {session}; + auto iter = m_expected_by_session.find(key); + if (iter == m_expected_by_session.end()) { + // First time we see this session: anchor on its first sequence. + m_expected_by_session.emplace(key, first_sequence + count); + m_messages_seen += count; + return 0; + } + + std::uint64_t& expected = iter->second; + std::uint64_t gap = 0; + if (first_sequence > expected) { + gap = first_sequence - expected; + m_gap_count += gap; + if (m_gap_callback) { + m_gap_callback(session, expected, first_sequence); + } + if (m_requester != nullptr) { + m_requester->request_retransmit(session, expected, gap); + } + } + + const std::uint64_t end_sequence = first_sequence + count; + if (end_sequence > expected) { + // Count only the genuinely new messages, not replayed duplicates. + const std::uint64_t already_seen = + expected > first_sequence ? expected - first_sequence : 0; + m_messages_seen += count - std::min(count, already_seen); + expected = end_sequence; + } + return gap; + } + + /// @brief Installs the gap-notification callback (empty clears it). + /// @param callback Invoked once per detected gap. + auto set_gap_callback(GapCallback callback) -> void { m_gap_callback = std::move(callback); } + + /// @brief Installs a non-owning retransmission hook (nullptr clears it). + /// @param requester Non-owning pointer to the requester to notify on + /// gaps, or nullptr to clear it. + auto set_retransmit_requester(RetransmitRequester* requester) noexcept -> void { + m_requester = requester; + } + + /// @brief The next sequence number expected for a session, if it has been seen. + /// @param session The session identifier to look up. + /// @return The next expected sequence number, or `std::nullopt` if the + /// session has not been observed. + [[nodiscard]] auto expected_next(std::string_view session + ) const -> std::optional { + const auto iter = m_expected_by_session.find(std::string {session}); + if (iter == m_expected_by_session.end()) { + return std::nullopt; + } + return iter->second; + } + + /// @brief Total number of missing messages detected across all sessions. + /// @return The total gap count. + [[nodiscard]] auto gap_count() const noexcept -> std::uint64_t { return m_gap_count; } + + /// @brief Total number of distinct sequenced messages observed. + /// @return The total count of distinct messages observed. + [[nodiscard]] auto messages_seen() const noexcept -> std::uint64_t { return m_messages_seen; } + + /// @brief Forgets all per-session state and resets the counters. + auto reset() -> void { + m_expected_by_session.clear(); + m_gap_count = 0; + m_messages_seen = 0; + } + + private: + std::unordered_map m_expected_by_session {}; + GapCallback m_gap_callback {}; + RetransmitRequester* m_requester {nullptr}; + std::uint64_t m_gap_count {0}; + std::uint64_t m_messages_seen {0}; +}; + +} // namespace itch::transport diff --git a/include/itch/transport/soupbintcp.hpp b/include/itch/transport/soupbintcp.hpp new file mode 100644 index 0000000..b80c790 --- /dev/null +++ b/include/itch/transport/soupbintcp.hpp @@ -0,0 +1,127 @@ +#pragma once + +/// @file +/// @brief Stateful decoder for the SoupBinTCP session-layer protocol. +/// +/// SoupBinTCP is NASDAQ's reliable, ordered TCP framing used for the Glimpse +/// snapshot service and for sequenced/unsequenced ITCH message delivery and +/// replay. This header declares the packet type enumeration and the +/// `SoupBinDecoder` that reassembles a raw byte stream into discrete packets +/// and decoded ITCH messages. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include +#include +#include +#include + +#include "itch/parser.hpp" +#include "itch/transport/sequencing.hpp" + +namespace itch::transport { + +/// @brief The SoupBinTCP packet types (the one-byte type that follows the +/// 2-byte length prefix). +/// +/// SoupBinTCP is the reliable, ordered TCP framing NASDAQ uses for the Glimpse +/// snapshot service and for recovery/replay. Only the server-to-client subset is +/// needed to consume a captured or replayed stream, but every defined type is +/// listed for completeness. +enum class SoupBinPacketType : char { + debug = '+', ///< Free-form debug text (either direction). + login_accepted = 'A', ///< Session id (10) + starting sequence number (20, ASCII). + login_rejected = 'J', ///< Reject reason code (1). + sequenced_data = 'S', ///< One sequenced application (ITCH) message. + server_heartbeat = 'H', ///< Keep-alive from the server. + end_of_session = 'Z', ///< The server has finished the session. + login_request = 'L', ///< Client login request. + unsequenced_data = 'U', ///< One unsequenced application message. + client_heartbeat = 'R', ///< Keep-alive from the client. + logout_request = 'O', ///< Client logout request. +}; + +/// @brief A stateful decoder for a SoupBinTCP byte stream. +/// +/// Because TCP delivers a byte stream rather than discrete packets, the decoder +/// accumulates input across `feed` calls and emits messages only for complete +/// packets. Sequenced (and, optionally, unsequenced) data packets each carry a +/// single ITCH message, which is decoded through an internal `Parser` and handed +/// to the `MessageCallback`. Control packets (login, heartbeat, logout, end of +/// session) are surfaced through the optional event callback. The starting +/// sequence number is learned from the Login Accepted packet and advanced per +/// sequenced message, with gaps reported through the embedded `SequenceTracker`. +class SoupBinDecoder { + public: + /// @brief Invoked for each non-data control packet, with its raw payload. + using EventCallback = + std::function payload)>; + + /// @brief Constructs a decoder that calls `callback` for each ITCH message. + /// + /// @param callback Invoked with each ITCH message decoded from a + /// sequenced or unsequenced data packet. + explicit SoupBinDecoder(MessageCallback callback); + + /// @brief Feeds a chunk of the TCP byte stream, processing any complete + /// packets it completes. + /// + /// @param bytes The next contiguous chunk of bytes read from the TCP + /// connection. + auto feed(std::span bytes) -> void; + + /// @brief Installs the control-packet event callback (empty clears it). + /// + /// @param callback Invoked for each non-data control packet decoded. + auto set_event_callback(EventCallback callback) -> void; + + /// @brief The embedded sequence tracker (install gap callbacks here). + /// @return Reference to the embedded `SequenceTracker`. + [[nodiscard]] auto tracker() noexcept -> SequenceTracker& { return m_tracker; } + /// @brief The embedded sequence tracker (install gap callbacks here). + /// @return Const reference to the embedded `SequenceTracker`. + [[nodiscard]] auto tracker() const noexcept -> const SequenceTracker& { return m_tracker; } + + /// @brief The session id learned from Login Accepted (empty until then). + /// @return The current session id, or an empty view before login. + [[nodiscard]] auto current_session() const -> std::string_view { return m_session; } + + /// @brief The next sequence number the decoder expects for a data packet. + /// @return The next expected sequence number. + [[nodiscard]] auto next_sequence() const noexcept -> std::uint64_t { return m_next_sequence; } + + /// @brief Total sequenced/unsequenced data messages decoded. + /// @return The total count of decoded data messages. + [[nodiscard]] auto messages_decoded() const noexcept -> std::uint64_t { + return m_messages_decoded; + } + + private: + /// @brief Decodes the single packet (type byte plus payload) and + /// dispatches it to the event callback or the application decoder. + /// + /// @param type The one-byte SoupBinTCP packet type. + /// @param payload The packet body following the type byte. + auto process_packet(SoupBinPacketType type, std::span payload) -> void; + + /// @brief Decodes one application message payload through the parser and + /// advances the expected sequence number. + /// + /// @param payload The raw ITCH message bytes carried by a sequenced or + /// unsequenced data packet. + auto decode_application_message(std::span payload) -> void; + + Parser m_parser {}; + MessageCallback m_callback; + EventCallback m_event_callback {}; + SequenceTracker m_tracker {}; + std::vector m_buffer {}; + std::string m_session {}; + std::uint64_t m_next_sequence {1}; + std::uint64_t m_messages_decoded {0}; +}; + +} // namespace itch::transport diff --git a/include/itch/venue.hpp b/include/itch/venue.hpp new file mode 100644 index 0000000..fea9f10 --- /dev/null +++ b/include/itch/venue.hpp @@ -0,0 +1,62 @@ +#pragma once + +/// @file +/// @brief The extension point for adding new ITCH-like venues/protocol versions +/// alongside NASDAQ TotalView-ITCH 5.0. +/// +/// Defines the `VenuePolicy` concept that a venue policy type must satisfy, and +/// `Nasdaq50`, the concrete policy for the only protocol this library +/// implements today. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include + +#include "itch/detail/wire.hpp" + +namespace itch::venue { + +/// @brief Concept describing a venue/protocol policy. +/// +/// ITCHCPP today implements NASDAQ TotalView-ITCH 5.0 concretely. This concept is +/// the extension seam for adding ITCH-like venues and versions (NASDAQ BX/PSX, +/// older ITCH 4.1, and other venue feeds) without rewriting the dispatch +/// machinery: a policy names itself and enumerates its message-type registry, and +/// the dispatch-table builder can be parameterized on it. +/// +/// A conforming policy provides: +/// - `static constexpr std::string_view name()` identifying the venue/version. +/// - `static void for_each_message_type(Visitor&&)` invoking +/// `visitor.template operator()(char)` once per message type, the same +/// shape as `itch::detail::for_each_message_type`. +template +concept VenuePolicy = requires { + { Policy::name() } -> std::convertible_to; +}; + +/// @brief The concrete policy for NASDAQ TotalView-ITCH 5.0 (the only venue +/// implemented today). New venues are added by writing another policy of +/// the same shape and parameterizing the dispatch builder on it. +struct Nasdaq50 { + /// @brief The identifying name of this venue/protocol version. + /// + /// @return The human-readable venue/version name. + [[nodiscard]] static constexpr auto name() noexcept -> std::string_view { + return "NASDAQ TotalView-ITCH 5.0"; + } + + /// @brief Enumerates this venue's message-type registry. + /// + /// @tparam Visitor The visitor type; must be callable as + /// `visitor.template operator()(char)`. + /// @param visitor The visitor invoked once per message type. + template + static constexpr auto for_each_message_type(Visitor&& visitor) -> void { + detail::for_each_message_type(static_cast(visitor)); + } +}; + +static_assert(VenuePolicy); + +} // namespace itch::venue diff --git a/packaging b/packaging new file mode 160000 index 0000000..115a19b --- /dev/null +++ b/packaging @@ -0,0 +1 @@ +Subproject commit 115a19b0306f80635bf2911e3b1d4a4ab409cfbf diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b9fb92c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,58 @@ +# Python packaging for the `itchcpp` wheel. + +[build-system] +requires = ["scikit-build-core>=0.8", "pybind11>=2.11"] +build-backend = "scikit_build_core.build" + +[project] +name = "itchcpp" +dynamic = ["version"] +description = "Native C++ NASDAQ TotalView-ITCH 5.0 parser, book engine, and analytics: a fast, drop-in backend for the pure-Python `itch` (itchfeed) package." +readme = "python/README.md" +requires-python = ">=3.9" +license = { text = "MIT" } +authors = [{ name = "Bertin Balouki Simyeli" }] +keywords = ["nasdaq", "itch", "market-data", "order-book", "hft"] +classifiers = [ + "Programming Language :: C++", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Office/Business :: Financial :: Investment", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] + +[project.urls] +Homepage = "https://github.com/bbalouki/itchcpp" +Documentation = "https://bbalouki.github.io/itchcpp/" + +[tool.scikit-build] +cmake.source-dir = "." +cmake.args = ["-DITCH_BUILD_PYTHON=ON", "-DITCH_CXX_STANDARD=20"] +cmake.define = { "CMAKE_VS_GLOBALS" = "IgnoreWarnIntDirInTempDetected=true" } +wheel.packages = ["python/itchcpp"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "VERSION.txt" +regex = "(?P.+)" + +[tool.cibuildwheel] +build = "cp39-* cp310-* cp311-* cp312-* cp313-*" +skip = "*-win32 *_i686 *-musllinux_*" +build-frontend = "build" + +manylinux-x86_64-image = "manylinux_2_28" +manylinux-aarch64-image = "manylinux_2_28" +test-requires = "pytest" +test-command = "pytest {project}/python/tests" + +[tool.cibuildwheel.macos.environment] +# C++20 with libc++ needs a recent deployment target. +MACOSX_DEPLOYMENT_TARGET = "11.0" diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt new file mode 100644 index 0000000..2e168e5 --- /dev/null +++ b/python/CMakeLists.txt @@ -0,0 +1,19 @@ +cmake_minimum_required(VERSION 3.25) + +# The Python bindings are built only when ITCH_BUILD_PYTHON is ON. pybind11 is +# resolved through vcpkg or a system install. +find_package(pybind11 CONFIG REQUIRED) + +# The compiled extension is the private `_itchcpp` module imported by the pure +# Python `itchcpp` package, which re-exports it under the upstream-compatible +# `itchcpp.messages` / `itchcpp.parser` / `itchcpp.indicators` layout. +pybind11_add_module(itchcpp_python src/bindings.cpp) +set_target_properties(itchcpp_python PROPERTIES OUTPUT_NAME "_itchcpp") + +target_link_libraries(itchcpp_python PRIVATE itch::itch) + +# When built through scikit-build-core (pip install .) install the extension into +# the itchcpp package directory alongside the pure-Python modules. +if(DEFINED SKBUILD) + install(TARGETS itchcpp_python LIBRARY DESTINATION itchcpp) +endif() diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..71a9a1b --- /dev/null +++ b/python/README.md @@ -0,0 +1,119 @@ +# itchcpp — a fast, drop-in backend for the pure-Python `itch` package + +`itchcpp` is a native (C++) NASDAQ TotalView-ITCH 5.0 parser, order-book engine, +and analytics layer exposed to Python via pybind11. It is built to be a +**drop-in, faster backend** for the pure-Python +[`itch`](https://github.com/bbalouki/itch) package (published on PyPI as +[`itchfeed`](https://pypi.org/project/itchfeed/)). + +The pure-Python parser unpacks every message with `struct.unpack` and materializes a +Python object in interpreter code, which is slow on full-day feeds. `itchcpp` does the +framing and field decoding in C++ (gigabytes per second) while exposing the **same +public API**: the same message classes (same names, same raw `bytes`/`int` +attributes), the same `MessageParser` semantics, the same `MarketMessage` helpers, +the same constants, and the same `create_message` factory. + +## Drop-in usage + +The package layout mirrors `itch.*`, so migrating only changes the import root: + +```python +# pure-Python # native backend +from itch.parser import MessageParser from itchcpp.parser import MessageParser +from itch.messages import create_message from itchcpp.messages import create_message +from itch.indicators import MARKET_CATEGORY from itchcpp.indicators import MARKET_CATEGORY +``` + +```python +from itchcpp.parser import MessageParser +from itchcpp.messages import AddOrderNoMPIAttributionMessage + +parser = MessageParser() # or MessageParser(message_type=b"AFE") to filter types +with open("01302020.NASDAQ_ITCH50", "rb") as itch_file: + for message in parser.parse_file(itch_file): + if isinstance(message, AddOrderNoMPIAttributionMessage): + # Raw attributes are bytes/int, exactly like the pure-Python package. + print(message.stock, message.decode_price("price"), message.shares) + # decode() returns a human-readable dataclass (symbols trimmed, + # prices scaled): + print(message.decode()) +``` + +`parse_file` takes an open **binary file object** (so `gzip.open(path, "rb")` works +too); `parse_stream(raw_bytes)` parses an in-memory buffer. Both return lazy +iterators, apply the `message_type` filter, and stop at the end-of-messages system +event (`S` with `event_code == b"C"`). + +## What is exposed + +- **`itchcpp.messages`** — every message class under its exact upstream name + (`SystemEventMessage`, `StockDirectoryMessage`, `AddOrderNoMPIAttributionMessage`, + `AddOrderMPIDAttribution`, `OrderExecutedMessage`, `NonCrossTradeMessage`, + `CrossTradeMessage`, `NOIIMessage`, `MWCBDeclineLeveMessage`, + `RetailPriceImprovementIndicator`, `DLCRMessage`, ...); the abstract bases + `MarketMessage`, `AddOrderMessage`, `ModifyOrderMessage`, `TradeMessage` (the + concrete classes are registered as virtual subclasses, so `isinstance` works); the + `messages` registry (`dict[bytes, type]`); the `AllMessages` constant; and the + `create_message(message_type, **kwargs)` factory. +- **`MarketMessage` helpers** on every message: `decode()`, `decode_price(name)`, + `to_bytes()`, `set_timestamp(ts1, ts2)`, `split_timestamp()`, + `get_attributes(call_able=False)`, plus the `message_type`, `description`, + `message_size`, and `price_precision` metadata. +- **`itchcpp.parser.MessageParser`** — `parse_file`, `parse_stream`, + `parse_messages(data, callback)`, `get_message_type(bytes)`, and the `message_type` + filter. +- **`itchcpp.indicators`** — the field code lookup tables (`SYSTEM_EVENT_CODES`, + `MARKET_CATEGORY`, `TRADING_STATES`, `ISSUE_SUB_TYPE_VALUES`, ...). +- **Native extras** (not part of the pure-Python API): `itchcpp.book` + (`BookManager`, `L3Book`, `Bbo`) for single-pass order-book reconstruction, and + `itchcpp.analytics` (`Vwap`). + +## Install + +Prebuilt wheels are published to PyPI for Linux, macOS, and Windows (CPython +3.9-3.13), so most users need no compiler: + +```bash +pip install itchcpp +``` + +### Build from source + +Building from source needs a C++20 compiler and pybind11 (pulled in automatically as +a build dependency). The `pyproject.toml` lives at the repository root, so build from +there: + +```bash +pip install . +``` + +This invokes scikit-build-core, which configures the CMake project with +`-DITCH_BUILD_PYTHON=ON` and builds the `itchcpp._itchcpp` extension module into the +`itchcpp` package. To build just the extension with CMake directly: + +```bash +cmake -S . -B build -DITCH_BUILD_PYTHON=ON +cmake --build build --target itchcpp_python +``` + +## Relationship to the `itch` (itchfeed) package + +`itchcpp` ships as a separate package and does not replace `itchfeed`; it exposes the +same API under the `itchcpp` import root. There are two migration paths: + +1. **Switch the import root** in your own code (`itch` -> `itchcpp`). +2. **Wire it as an optional backend** in `itchfeed`, so installed users get a + transparent speedup with no code change: + + ```python + try: + from itchcpp.parser import MessageParser # fast C++ backend + from itchcpp.messages import create_message, messages + except ImportError: # fall back to the pure-Python implementation + from itch.parser import MessageParser + from itch.messages import create_message, messages + ``` + +The native message objects expose **raw** field values (`bytes` for character fields, +`int` for prices and quantities) just like the pure-Python classes, so existing code +that calls `decode_price(...)` or `decode()` keeps working unchanged. diff --git a/python/itchcpp/__init__.py b/python/itchcpp/__init__.py new file mode 100644 index 0000000..7e6ba6a --- /dev/null +++ b/python/itchcpp/__init__.py @@ -0,0 +1,101 @@ +"""itchcpp: a native (C++) NASDAQ TotalView-ITCH 5.0 backend. + +``itchcpp`` is a fast, drop-in backend for the pure-Python ``itch`` (PyPI: +``itchfeed``) package. It mirrors that package's public API so existing code only +needs to change the import root, e.g.:: + + from itch.parser import MessageParser # pure-Python + from itchcpp.parser import MessageParser # native backend + +The same message classes, ``MessageParser`` semantics, ``MarketMessage`` helpers +(``decode`` / ``decode_price`` / ``to_bytes`` / ``set_timestamp`` / +``split_timestamp`` / ``get_attributes``), the ``messages`` registry, the +``AllMessages`` constant, ``create_message``, and the :mod:`itchcpp.indicators` +tables are provided. The :mod:`itchcpp.book` and :mod:`itchcpp.analytics` submodules +add native extras (order-book reconstruction and VWAP) that are not part of the +pure-Python API. +""" + +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version + +from . import analytics, book, indicators, messages, parser +from .messages import ( + AddOrderMessage, + AddOrderMPIDAttribution, + AddOrderNoMPIAttributionMessage, + AllMessages, + BrokenTradeMessage, + CrossTradeMessage, + DLCRMessage, + IPOQuotingPeriodUpdateMessage, + LULDAuctionCollarMessage, + MarketMessage, + MarketParticipantPositionMessage, + ModifyOrderMessage, + MWCBDeclineLeveMessage, + MWCBStatusMessage, + NOIIMessage, + NonCrossTradeMessage, + OperationalHaltMessage, + OrderCancelMessage, + OrderDeleteMessage, + OrderExecutedMessage, + OrderExecutedWithPriceMessage, + OrderReplaceMessage, + RegSHOMessage, + RetailPriceImprovementIndicator, + StockDirectoryMessage, + StockTradingActionMessage, + SystemEventMessage, + TradeMessage, + create_message, + messages as message_registry, +) +from .parser import MessageParser + +try: + __version__ = version("itchcpp") +except PackageNotFoundError: # pragma: no cover - source checkout without install + __version__ = "unknown" + +__all__ = [ + "AddOrderMPIDAttribution", + "AddOrderMessage", + "AddOrderNoMPIAttributionMessage", + "AllMessages", + "BrokenTradeMessage", + "CrossTradeMessage", + "DLCRMessage", + "IPOQuotingPeriodUpdateMessage", + "LULDAuctionCollarMessage", + "MWCBDeclineLeveMessage", + "MWCBStatusMessage", + "MarketMessage", + "MarketParticipantPositionMessage", + "MessageParser", + "ModifyOrderMessage", + "NOIIMessage", + "NonCrossTradeMessage", + "OperationalHaltMessage", + "OrderCancelMessage", + "OrderDeleteMessage", + "OrderExecutedMessage", + "OrderExecutedWithPriceMessage", + "OrderReplaceMessage", + "RegSHOMessage", + "RetailPriceImprovementIndicator", + "StockDirectoryMessage", + "StockTradingActionMessage", + "SystemEventMessage", + "TradeMessage", + "__version__", + "analytics", + "book", + "create_message", + "indicators", + "message_registry", + "messages", + "parser", +] diff --git a/python/itchcpp/_bases.py b/python/itchcpp/_bases.py new file mode 100644 index 0000000..f9e0a94 --- /dev/null +++ b/python/itchcpp/_bases.py @@ -0,0 +1,116 @@ +"""Abstract base classes for the ITCH message hierarchy. + +These mirror the bases in ``itch.messages`` from the pure-Python ``itchfeed`` +package. The concrete message classes are implemented in the compiled ``_itchcpp`` +extension and registered as virtual subclasses of these bases (so ``isinstance`` +checks behave as upstream), which is why the bases live in their own dependency-free +module: it lets the type stubs reference them without an import cycle. + +The method bodies match the upstream :class:`MarketMessage` and are used only when a +base is instantiated directly; the native classes carry their own C++ implementations. +""" + +from __future__ import annotations + +from abc import ABCMeta +from dataclasses import make_dataclass +from typing import TYPE_CHECKING, Any, List, Tuple, overload + + +class MarketMessage(metaclass=ABCMeta): + """Abstract base for every ITCH message.""" + + message_type: bytes + description: str + message_size: int + price_precision: int = 4 + timestamp: int + stock_locate: int + tracking_number: int + + if TYPE_CHECKING: + # The concrete (native) subclasses are default-constructible and can also + # be built from a raw message body; these overloads are for type-checkers + # only and add no runtime __init__ to the abstract base. + @overload + def __init__(self) -> None: ... + @overload + def __init__(self, body: bytes) -> None: ... + + def __repr__(self) -> str: + return repr(self.decode()) + + def __bytes__(self) -> bytes: + return self.to_bytes() + + def to_bytes(self) -> bytes: # pragma: no cover - overridden by concrete classes + raise NotImplementedError + + def set_timestamp(self, ts1: int, ts2: int) -> None: + self.timestamp = ts2 | (ts1 << 32) + + def split_timestamp(self) -> Tuple[int, int]: + ts1 = self.timestamp >> 32 + ts2 = self.timestamp - (ts1 << 32) + return (ts1, ts2) + + def decode_price(self, price_attr: str) -> float: + price = getattr(self, price_attr) + if callable(price): + raise ValueError(f"Please check the price attribute for {price_attr}") + return price / (10 ** getattr(self, "price_precision")) + + def decode(self, prefix: str = "") -> Any: + builtin_attrs = {} + for attr in dir(self): + if attr.startswith("__"): + continue + value = getattr(self, attr) + if "price" in attr and attr not in ("price_precision", "decode_price"): + value = self.decode_price(attr) + if callable(value): + continue + if isinstance(value, (int, float, str, bool)): + builtin_attrs[attr] = value + elif isinstance(value, bytes): + try: + builtin_attrs[attr] = value.decode(encoding="ascii").rstrip() + except UnicodeDecodeError: + builtin_attrs[attr] = value + fields = [(key, type(value)) for key, value in builtin_attrs.items()] + decoded_class = make_dataclass(f"{prefix}{self.__class__.__name__}", fields) + return decoded_class(**builtin_attrs) + + def get_attributes(self, call_able: bool = False) -> List[str]: + attrs = [attr for attr in dir(self) if not attr.startswith("__")] + return [attr for attr in attrs if callable(getattr(self, attr)) == call_able] + + +class AddOrderMessage(MarketMessage, metaclass=ABCMeta): + """Abstract base for the two Add Order message variants.""" + + order_reference_number: int + buy_sell_indicator: bytes + shares: int + stock: bytes + price: int + + +class ModifyOrderMessage(MarketMessage, metaclass=ABCMeta): + """Abstract base for messages that modify a resting order.""" + + order_reference_number: int + + +class TradeMessage(MarketMessage, metaclass=ABCMeta): + """Abstract base for trade messages.""" + + match_number: int + + +__all__ = [ + "AddOrderMessage", + "MarketMessage", + "ModifyOrderMessage", + "TradeMessage", +] diff --git a/python/itchcpp/_itchcpp.pyi b/python/itchcpp/_itchcpp.pyi new file mode 100644 index 0000000..de87da6 --- /dev/null +++ b/python/itchcpp/_itchcpp.pyi @@ -0,0 +1,188 @@ +"""Type stubs for the compiled ``itchcpp._itchcpp`` extension module. + +The concrete message classes are implemented in C++. For typing they are presented +as subclasses of the abstract bases in :mod:`itchcpp._bases` (``MarketMessage`` and +friends), matching the runtime ``isinstance`` behavior established by virtual-subclass +registration in :mod:`itchcpp.messages`. +""" + +from typing import Any, BinaryIO, Callable, Iterator + +from ._bases import ( + AddOrderMessage, + MarketMessage, + ModifyOrderMessage, + TradeMessage, +) + +AllMessages: bytes +messages: dict[bytes, type[MarketMessage]] + +class SystemEventMessage(MarketMessage): + event_code: bytes + +class StockDirectoryMessage(MarketMessage): + stock: bytes + market_category: bytes + financial_status_indicator: bytes + round_lot_size: int + round_lots_only: bytes + issue_classification: bytes + issue_sub_type: bytes + authenticity: bytes + short_sale_threshold_indicator: bytes + ipo_flag: bytes + luld_ref: bytes + etp_flag: bytes + etp_leverage_factor: int + inverse_indicator: bytes + +class StockTradingActionMessage(MarketMessage): + stock: bytes + trading_state: bytes + reserved: bytes + reason: bytes + +class RegSHOMessage(MarketMessage): + stock: bytes + reg_sho_action: bytes + +class MarketParticipantPositionMessage(MarketMessage): + mpid: bytes + stock: bytes + primary_market_maker: bytes + market_maker_mode: bytes + market_participant_state: bytes + +class MWCBDeclineLeveMessage(MarketMessage): + level1_price: int + level2_price: int + level3_price: int + +class MWCBStatusMessage(MarketMessage): + breached_level: bytes + +class IPOQuotingPeriodUpdateMessage(MarketMessage): + stock: bytes + ipo_release_time: int + ipo_release_qualifier: bytes + ipo_price: int + +class LULDAuctionCollarMessage(MarketMessage): + stock: bytes + auction_collar_reference_price: int + upper_auction_collar_price: int + lower_auction_collar_price: int + auction_collar_extention: int + +class OperationalHaltMessage(MarketMessage): + stock: bytes + market_code: bytes + operational_halt_action: bytes + +class AddOrderNoMPIAttributionMessage(AddOrderMessage): ... + +class AddOrderMPIDAttribution(AddOrderMessage): + attribution: bytes + +class OrderExecutedMessage(ModifyOrderMessage): + executed_shares: int + match_number: int + +class OrderExecutedWithPriceMessage(ModifyOrderMessage): + executed_shares: int + match_number: int + printable: bytes + execution_price: int + +class OrderCancelMessage(ModifyOrderMessage): + cancelled_shares: int + +class OrderDeleteMessage(ModifyOrderMessage): ... + +class OrderReplaceMessage(ModifyOrderMessage): + new_order_reference_number: int + shares: int + price: int + +class NonCrossTradeMessage(TradeMessage): + order_reference_number: int + buy_sell_indicator: bytes + shares: int + stock: bytes + price: int + +class CrossTradeMessage(TradeMessage): + shares: int + stock: bytes + cross_price: int + cross_type: bytes + +class BrokenTradeMessage(TradeMessage): ... + +class NOIIMessage(MarketMessage): + paired_shares: int + imbalance_shares: int + imbalance_direction: bytes + stock: bytes + far_price: int + near_price: int + current_reference_price: int + cross_type: bytes + variation_indicator: bytes + +class RetailPriceImprovementIndicator(MarketMessage): + stock: bytes + interest_flag: bytes + +class DLCRMessage(MarketMessage): + stock: bytes + open_eligibility_status: bytes + minimum_allowable_price: int + maximum_allowable_price: int + near_execution_price: int + near_execution_time: int + lower_price_range_collar: int + upper_price_range_collar: int + +def create_message(message_type: bytes, **kwargs: Any) -> MarketMessage: ... + +class MessageParser: + message_type: bytes + def __init__(self, message_type: bytes = ...) -> None: ... + def get_message_type(self, message: bytes) -> MarketMessage: ... + def parse_stream( + self, data: bytes, save_file: BinaryIO | None = ... + ) -> Iterator[MarketMessage]: ... + def parse_file( + self, file: BinaryIO, cachesize: int = ..., save_file: BinaryIO | None = ... + ) -> Iterator[MarketMessage]: ... + def parse_messages( + self, data: bytes | BinaryIO, callback: Callable[[MarketMessage], None] + ) -> None: ... + +class Bbo: + has_bid: bool + has_ask: bool + bid_shares: int + ask_shares: int + bid_price: float + ask_price: float + +class L3Book: + symbol: str + def bbo(self) -> Bbo: ... + +class BookManager: + def __init__(self) -> None: ... + def track_symbol(self, symbol: str) -> None: ... + def process_stream(self, raw_binary_data: bytes) -> None: ... + def book_count(self) -> int: ... + def book_for_symbol(self, symbol: str) -> L3Book: ... + +class Vwap: + def __init__(self) -> None: ... + def add(self, raw_price: int, shares: int) -> None: ... + def value(self) -> float: ... + def volume(self) -> int: ... + def reset(self) -> None: ... diff --git a/python/itchcpp/analytics.py b/python/itchcpp/analytics.py new file mode 100644 index 0000000..835c531 --- /dev/null +++ b/python/itchcpp/analytics.py @@ -0,0 +1,9 @@ +"""Native streaming analytics (an itchcpp extra, beyond the pure-Python ``itch`` API). + +Currently exposes a volume-weighted average price (VWAP) accumulator fed with raw +4-decimal prices and share quantities. +""" + +from ._itchcpp import Vwap + +__all__ = ["Vwap"] diff --git a/python/itchcpp/book.py b/python/itchcpp/book.py new file mode 100644 index 0000000..4d3b1a2 --- /dev/null +++ b/python/itchcpp/book.py @@ -0,0 +1,9 @@ +"""Native order-book engine (an itchcpp extra, beyond the pure-Python ``itch`` API). + +Reconstructs per-symbol L3 books and best bid/offer directly from an ITCH stream +in a single C++ pass. +""" + +from ._itchcpp import Bbo, BookManager, L3Book + +__all__ = ["Bbo", "BookManager", "L3Book"] diff --git a/python/itchcpp/indicators.py b/python/itchcpp/indicators.py new file mode 100644 index 0000000..852e5bd --- /dev/null +++ b/python/itchcpp/indicators.py @@ -0,0 +1,213 @@ +"""Indicator/code lookup tables for ITCH 5.0 fields. + +Vendored verbatim from ``itch.indicators`` in the pure-Python ``itchfeed`` package +so that code reading these maps works unchanged against the native backend. Each +table maps a single- or multi-byte code to its human-readable description. +""" + +SYSTEM_EVENT_CODES = { + b"O": "Start of Messages", + b"S": "Start of System hours", + b"Q": "Start of Market hours", + b"M": "End of Market hours", + b"E": "End of System hours", + b"C": "End of Messages", +} + + +MARKET_CATEGORY = { + b"N": "NYSE", + b"A": "AMEX", + b"P": "Arca", + b"Q": "NASDAQ Global Select", + b"G": "NASDAQ Global Market", + b"S": "NASDAQ Capital Market", + b"Z": "BATS", + b"V": "Investors Exchange", + b" ": "Not available", +} + + +FINANCIAL_STATUS_INDICATOR = { + b"D": "Deficient", + b"E": "Delinquent", + b"Q": "Bankrupt", + b"S": "Suspended", + b"G": "Deficient and Bankrupt", + b"H": "Deficient and Delinquent", + b"J": "Delinquent and Bankrupt", + b"K": "Deficient, Delinquent and Bankrupt", + b"C": "Creations and/or Redemptions Suspended for Exchange Traded Product", + b"N": "Normal (Default): Issuer is NOT Deficient, Delinquent, or Bankrupt", + b" ": "Not available. Firms should refer to SIAC feeds for code if needed", +} + + +ISSUE_CLASSIFICATION_VALUES = { + b"A": "American Depositary Share", + b"B": "Bond", + b"C": "Common Stock", + b"F": "Depository Receipt", + b"I": "144A", + b"L": "Limited Partnership", + b"N": "Notes", + b"O": "Ordinary Share", + b"P": "Preferred Stock", + b"Q": "Other Securities", + b"R": "Right", + b"S": "Shares of Beneficial Interest", + b"T": "Convertible Debenture", + b"U": "Unit", + b"V": "Units/Beneficial Interest", + b"W": "Warrant", +} + + +ISSUE_SUB_TYPE_VALUES = { + b"A": "Preferred Trust Securities", + b"AI": "Alpha Index ETNs", + b"B": "Index Based Derivative", + b"C": "Common Shares", + b"CB": "Commodity Based Trust Shares", + b"CF": "Commodity Futures Trust Shares", + b"CL": "Commodity-Linked Securities", + b"CM": "Commodity Index Trust Shares", + b"CO": "Collateralized Mortgage Obligation", + b"CT": "Currency Trust Shares", + b"CU": "Commodity-Currency-Linked Securities", + b"CW": "Currency Warrants", + b"D": "Global Depositary Shares", + b"E": "ETF-Portfolio Depositary Receipt", + b"EG": "Equity Gold Shares", + b"EI": "ETN-Equity Index-Linked Securities", + b"EM": "NextShares Exchange Traded Managed Fund*", + b"EN": "Exchange Traded Notes", + b"EU": "Equity Units", + b"F": "HOLDRS", + b"FI": "ETN-Fixed Income-Linked Securities", + b"FL": "ETN-Futures-Linked Securities", + b"G": "Global Shares", + b"I": "ETF-Index Fund Shares", + b"IR": "Interest Rate", + b"IW": "Index Warrant", + b"IX": "Index-Linked Exchangeable Notes", + b"J": "Corporate Backed Trust Security", + b"L": "Contingent Litigation Right", + b"LL": "Limited Liability Company (LLC)", + b"M": "Equity-Based Derivative", + b"MF": "Managed Fund Shares", + b"ML": "ETN-Multi-Factor Index-Linked Securities", + b"MT": "Managed Trust Securities", + b"N": "NY Registry Shares", + b"O": "Open Ended Mutual Fund", + b"P": "Privately Held Security", + b"PP": "Poison Pill", + b"PU": "Partnership Units", + b"Q": "Closed-End Funds", + b"R": "Reg-S", + b"RC": "Commodity-Redeemable Commodity-Linked Securities", + b"RF": "ETN-Redeemable Futures-Linked Securities", + b"RT": "REIT", + b"RU": "Commodity-Redeemable Currency-Linked Securities", + b"S": "SEED", + b"SC": "Spot Rate Closing", + b"SI": "Spot Rate Intraday", + b"T": "Tracking Stock", + b"TC": "Trust Certificates", + b"TU": "Trust Units", + b"U": "Portal", + b"V": "Contingent Value Right", + b"W": "Trust Issued Receipts", + b"WC": "World Currency Option", + b"X": "Trust", + b"Y": "Other", + b"Z": "Not Applicable", +} + + +TRADING_STATES = { + b"H": "Halted across all U.S. equity markets / SROs", + b"P": "Paused across all U.S. equity markets / SROs", + b"Q": "Quotation only period for cross-SRO halt or pause", + b"T": "Trading on NASDAQ", +} + + +TRADING_ACTION_REASON_CODES = { + b"T1": "Halt News Pending", + b"T2": "Halt News Disseminated", + b"T3": "News and Resumption Times", + b"T5": "Single Security Trading Pause In Effect", + b"T6": "Regulatory Halt - Extraordinary Market Activity", + b"T7": "Single Security Trading Pause / Quotation Only Period", + b"T8": "Halt ETF", + b"T12": "Trading Halted; For Information Requested by Listing Market", + b"H4": "Halt Non-Compliance", + b"H9": "Halt Filings Not Current", + b"H10": "Halt SEC Trading Suspension", + b"H11": "Halt Regulatory Concern", + b"O1": "Operations Halt; Contact Market Operations", + b"LUDP": "Volatility Trading Pause", + b"LUDS": "Volatility Trading Pause - Straddle Condition", + b"MWC0": "Market Wide Circuit Breaker Halt - Carry over from previous day", + b"MWC1": "Market Wide Circuit Breaker Halt - Level 1", + b"MWC2": "Market Wide Circuit Breaker Halt - Level 2", + b"MWC3": "Market Wide Circuit Breaker Halt - Level 3", + b"MWCQ": "Market Wide Circuit Breaker Resumption", + b"IPO1": "IPO Issue Not Yet Trading", + b"IPOQ": "IPO Security Released for Quotation (Nasdaq Securities Only)", + b"IPOE": "IPO Security — Positioning Window Extension (Nasdaq Securities Only)", + b"M1": "Corporate Action", + b"M2": "Quotation Not Available", + b"R1": "New Issue Available", + b"R2": "Issue Available", + b"R4": "Qualifications Issues Reviewed/Resolved; Quotations/Trading to Resume", + b"R9": "Filing Requirements Satisfied/Resolved; Quotations/Trading To Resume", + b"C3": "Issuer News Not Forthcoming; Quotations/Trading To Resume", + b"C4": "Qualifications Halt Ended; Maintenance Requirements Met; Resume", + b"C9": "Qualifications Halt Concluded; Filings Met; Quotes/Trades To Resume", + b"C11": "Trade Halt Concluded By Other Regulatory Authority; Quotes/Trades Resume", + b" ": "Reason Not Available", +} + + +PRIMARY_MARKET_MAKER = { + b"Y": "Primary market maker", + b"N": "Non-primary market maker", +} + + +MARKET_MAKER_MODE = { + b"N": "Normal", + b"P": "Passive", + b"S": "Syndicate", + b"R": "Pre-syndicate", + b"L": "Penalty", +} + + +MARKET_PARTICIPANT_STATE = { + b"A": "Active", + b"E": "Excused", + b"W": "Withdrawn", + b"S": "Suspended", + b"D": "Deleted", +} + + +PRICE_VARIATION_INDICATOR = { + b"L": "Less than 1%", + b"1": "1 to 1.99%", + b"2": "2 to 2.99%", + b"3": "3 to 3.99%", + b"4": "4 to 4.99%", + b"5": "5 to 5.99%", + b"6": "6 to 6.99%", + b"7": "7 to 7.99%", + b"8": "8 to 8.99%", + b"9": "9 to 9.99%", + b"A": "10 to 19.99%", + b"B": "20 to 29.99%", + b"C": "30% or greater", + b" ": "Cannot be calculated", +} diff --git a/python/itchcpp/messages.py b/python/itchcpp/messages.py new file mode 100644 index 0000000..a9d628f --- /dev/null +++ b/python/itchcpp/messages.py @@ -0,0 +1,102 @@ +"""Message classes for the native ITCH backend. + +Mirrors ``itch.messages`` from the pure-Python ``itchfeed`` package: the same +concrete message classes (re-exported from the compiled ``_itchcpp`` extension), +the ``messages`` registry, the ``AllMessages`` constant, and the ``create_message`` +factory. The abstract base classes (:class:`MarketMessage`, :class:`AddOrderMessage`, +:class:`ModifyOrderMessage`, :class:`TradeMessage`) come from :mod:`itchcpp._bases` +and the native classes are registered here as virtual subclasses so ``isinstance`` +checks behave exactly as they do upstream. +""" + +from __future__ import annotations + +from ._bases import ( + AddOrderMessage, + MarketMessage, + ModifyOrderMessage, + TradeMessage, +) +from ._itchcpp import ( # noqa: F401 (re-exported) + AddOrderMPIDAttribution, + AddOrderNoMPIAttributionMessage, + AllMessages, + BrokenTradeMessage, + CrossTradeMessage, + DLCRMessage, + IPOQuotingPeriodUpdateMessage, + LULDAuctionCollarMessage, + MarketParticipantPositionMessage, + MWCBDeclineLeveMessage, + MWCBStatusMessage, + NOIIMessage, + NonCrossTradeMessage, + OperationalHaltMessage, + OrderCancelMessage, + OrderDeleteMessage, + OrderExecutedMessage, + OrderExecutedWithPriceMessage, + OrderReplaceMessage, + RegSHOMessage, + RetailPriceImprovementIndicator, + StockDirectoryMessage, + StockTradingActionMessage, + SystemEventMessage, + create_message, + messages, +) + + +# Register the native classes as virtual subclasses for isinstance/issubclass parity. +_ADD_ORDER = (AddOrderNoMPIAttributionMessage, AddOrderMPIDAttribution) +_MODIFY_ORDER = ( + OrderExecutedMessage, + OrderExecutedWithPriceMessage, + OrderCancelMessage, + OrderDeleteMessage, + OrderReplaceMessage, +) +_TRADE = (NonCrossTradeMessage, CrossTradeMessage, BrokenTradeMessage) + +for _cls in messages.values(): + MarketMessage.register(_cls) +for _cls in _ADD_ORDER: + AddOrderMessage.register(_cls) +for _cls in _MODIFY_ORDER: + ModifyOrderMessage.register(_cls) +for _cls in _TRADE: + TradeMessage.register(_cls) + + +__all__ = [ + "AddOrderMessage", + "AddOrderMPIDAttribution", + "AddOrderNoMPIAttributionMessage", + "AllMessages", + "BrokenTradeMessage", + "CrossTradeMessage", + "DLCRMessage", + "IPOQuotingPeriodUpdateMessage", + "LULDAuctionCollarMessage", + "MarketMessage", + "MarketParticipantPositionMessage", + "ModifyOrderMessage", + "MWCBDeclineLeveMessage", + "MWCBStatusMessage", + "NOIIMessage", + "NonCrossTradeMessage", + "OperationalHaltMessage", + "OrderCancelMessage", + "OrderDeleteMessage", + "OrderExecutedMessage", + "OrderExecutedWithPriceMessage", + "OrderReplaceMessage", + "RegSHOMessage", + "RetailPriceImprovementIndicator", + "StockDirectoryMessage", + "StockTradingActionMessage", + "SystemEventMessage", + "TradeMessage", + "create_message", + "messages", +] diff --git a/python/itchcpp/parser.py b/python/itchcpp/parser.py new file mode 100644 index 0000000..1f1b62e --- /dev/null +++ b/python/itchcpp/parser.py @@ -0,0 +1,12 @@ +"""Native ITCH message parser. + +Mirrors ``itch.parser`` from the pure-Python ``itchfeed`` package. The +:class:`MessageParser` is implemented in C++ (the ``_itchcpp`` extension) and +exposes the same interface: an optional ``message_type`` byte filter, lazy +``parse_stream`` / ``parse_file`` iterators, ``parse_messages`` callback dispatch, +and ``get_message_type``. +""" + +from ._itchcpp import AllMessages, MessageParser + +__all__ = ["AllMessages", "MessageParser"] diff --git a/python/itchcpp/py.typed b/python/itchcpp/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/python/src/bindings.cpp b/python/src/bindings.cpp new file mode 100644 index 0000000..51d1513 --- /dev/null +++ b/python/src/bindings.cpp @@ -0,0 +1,863 @@ +// Native (C++) backend for the pure-Python `itch` (PyPI: `itchfeed`) package. +// +// This module, compiled as `itchcpp._itchcpp`, mirrors the upstream public API +// exactly so it can serve as a faster drop-in: the same message classes (same +// names, same raw `bytes`/`int` attributes), the same `MessageParser` semantics +// (a `message_type` filter, lazy iteration, stop on the end-of-messages system +// event), the same `MarketMessage` helpers (`decode`, `decode_price`, +// `get_attributes`, `set_timestamp`, `split_timestamp`, `to_bytes`), and the same +// module members (`AllMessages`, `messages`, `create_message`). The pure-Python +// `itchcpp` package layered on top re-exports these under `itchcpp.messages`, +// `itchcpp.parser`, and `itchcpp.indicators`. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/analytics/vwap.hpp" +#include "itch/book/book_manager.hpp" +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/parser.hpp" + +namespace py = pybind11; + +namespace { + +// The set of every known message-type byte, in the upstream order. Mirrors +// `itch.messages.AllMessages` and doubles as the default parser filter. +constexpr std::string_view ALL_MESSAGES = "SAFECXDUBHRYPQINLVWKJhO"; + +// The width of the 2-byte big-endian length prefix that frames each message. +constexpr std::size_t FRAME_HEADER_LEN = 2; + +// Default chunk size for reading a file-backed stream (matches upstream's 64 KiB). +constexpr std::size_t DEFAULT_CACHE_SIZE = 65536; + +// Generic message helpers +// These operate on the Python object (`self`) so a single implementation serves +// every message class, exactly replicating the upstream `MarketMessage` methods. + +// Returns whether a looked-up attribute value is callable (a bound method). +auto is_callable(const py::object& value) -> bool { return PyCallable_Check(value.ptr()) != 0; } + +// decode_price(name): divide the named raw integer field by 10**price_precision, +// matching `MarketMessage.decode_price`. Raises ValueError for a method name and +// (via Python) TypeError when the attribute is not numeric. +auto message_decode_price(const py::object& self, const std::string& price_attr) -> py::object { + const py::object value = self.attr(price_attr.c_str()); + if (is_callable(value)) { + throw py::value_error("Please check the price attribute for " + price_attr); + } + const py::object precision = self.attr("price_precision"); + const py::object divisor = py::int_(10).attr("__pow__")(precision); + return value / divisor; +} + +// set_timestamp(ts1, ts2): reconstruct the 48-bit timestamp from two 32-bit halves. +auto message_set_timestamp(const py::object& self, std::uint64_t ts1, std::uint64_t ts2) -> void { + self.attr("timestamp") = py::int_(ts2 | (ts1 << 32U)); +} + +// split_timestamp(): split the timestamp into (high 32 bits, low 32 bits). +auto message_split_timestamp(const py::object& self) -> py::tuple { + const auto timestamp = self.attr("timestamp").cast(); + const std::uint64_t high {timestamp >> 32U}; + const std::uint64_t low {timestamp - (high << 32U)}; + return py::make_tuple(high, low); +} + +// get_attributes(call_able): the non-callable (data) or callable (method) public +// attribute names, derived from dir(self) just like the upstream method. +auto message_get_attributes(const py::object& self, bool call_able) -> py::list { + py::list out; + const py::object names = py::module_::import("builtins").attr("dir")(self); + for (const auto& name : names) { + const auto text = name.cast(); + if (text.rfind("__", 0) == 0) { + continue; + } + if (is_callable(self.attr(name)) == call_able) { + out.append(name); + } + } + return out; +} + +// decode(prefix): build a human-readable dataclass named `{prefix}{ClassName}`, +// turning bytes fields into rstripped ASCII strings and price fields into floats. +auto message_decode(const py::object& self, const std::string& prefix) -> py::object { + const py::object builtins = py::module_::import("builtins"); + py::dict attrs; + for (const auto& name : builtins.attr("dir")(self)) { + const auto text = name.cast(); + if (text.rfind("__", 0) == 0) { + continue; + } + py::object value = self.attr(name); + if (text.find("price") != std::string::npos && text != "price_precision" && + text != "decode_price") { + value = self.attr("decode_price")(name); + } + if (is_callable(value)) { + continue; + } + if (py::isinstance(value)) { + attrs[name] = value.attr("decode")("ascii").attr("rstrip")(); + } else if ( + py::isinstance(value) || py::isinstance(value) || + py::isinstance(value) || py::isinstance(value) + ) { + attrs[name] = value; + } + } + + py::list fields; + for (const auto& item : attrs) { + fields.append(py::make_tuple(item.first, builtins.attr("type")(item.second))); + } + const auto class_name = prefix + self.attr("__class__").attr("__name__").cast(); + const py::object make_dataclass = py::module_::import("dataclasses").attr("make_dataclass"); + return make_dataclass(class_name, fields)(**attrs); +} + +// Field exposure helpers + +// Exposes a single-character field as a length-1 `bytes` value (read/write). +template +auto def_byte(py::class_& cls, const char* name, char MsgType::* member) -> void { + cls.def_property( + name, + [member](const MsgType& msg) { return py::bytes(&(msg.*member), 1); }, + [member](MsgType& msg, const py::bytes& value) { + const std::string text {value}; + msg.*member = text.empty() ? '\0' : text.front(); + } + ); +} + +// Exposes a fixed-width character array as a `bytes` value of that width (the raw, +// space-padded symbol, not trimmed), matching upstream attribute semantics. +template +auto def_byte_array(py::class_& cls, const char* name, char (MsgType::*member)[Width]) + -> void { + cls.def_property( + name, + [member](const MsgType& msg) { return py::bytes(msg.*member, Width); }, + [member](MsgType& msg, const py::bytes& value) { + const std::string text {value}; + std::fill(std::begin(msg.*member), std::end(msg.*member), '\0'); + std::memcpy(msg.*member, text.data(), std::min(text.size(), Width)); + } + ); +} + +// Serializes a message back to its wire body (type byte first, no frame length). +template +auto message_to_bytes(const MsgType& msg) -> py::bytes { + const std::vector bytes = itch::encode_message(itch::Message {msg}); + return {reinterpret_cast(bytes.data()), bytes.size()}; // NOLINT +} + +// Frames a body so the core parser (which expects a 2-byte big-endian length) can +// decode it: for every real ITCH message (< 256 bytes) this is byte-identical to +// the upstream `\x00` + length-byte header. +auto unpack_body(const std::string& body) -> itch::Message { + std::string frame; + frame.reserve(FRAME_HEADER_LEN + body.size()); + frame.push_back(static_cast((body.size() >> 8U) & 0xFFU)); + frame.push_back(static_cast(body.size() & 0xFFU)); + frame.append(body); + + itch::Parser parser; + std::optional result; + parser.parse(frame.data(), frame.size(), [&result](const itch::Message& message) { + result = message; + }); + if (!result.has_value()) { + throw py::value_error("Could not unpack message body"); + } + return *result; +} + +// Casts a parsed variant alternative to its concrete Python message object. +auto message_to_object(const itch::Message& message) -> py::object { + return std::visit([](const auto& concrete) { return py::cast(concrete); }, message); +} + +// Binds the fields and helpers common to every message class. `precision` is 4 for +// all messages except the MWCB decline levels (8). Returns the class so the caller +// can append its message-specific fields. +template +auto bind_common(py::module_& mod, const char* name, const char* description, int precision) + -> py::class_ { + py::class_ cls {mod, name}; + cls.def(py::init<>()); + cls.def(py::init([](const py::bytes& body) { + return std::get(unpack_body(std::string {body})); + })); + + cls.def_property_readonly("message_type", [](const MsgType& msg) { + return py::bytes(&msg.message_type, 1); + }); + cls.def_readwrite("stock_locate", &MsgType::stock_locate); + cls.def_readwrite("tracking_number", &MsgType::tracking_number); + cls.def_readwrite("timestamp", &MsgType::timestamp); + + cls.def("decode_price", &message_decode_price, py::arg("price_attr")); + cls.def("set_timestamp", &message_set_timestamp, py::arg("ts1"), py::arg("ts2")); + cls.def("split_timestamp", &message_split_timestamp); + cls.def("get_attributes", &message_get_attributes, py::arg("call_able") = false); + cls.def("decode", &message_decode, py::arg("prefix") = std::string {}); + cls.def("to_bytes", &message_to_bytes); + cls.def("__bytes__", &message_to_bytes); + cls.def("__repr__", [](const py::object& self) { return py::repr(self.attr("decode")()); }); + + cls.attr("description") = py::str(description); + cls.attr("price_precision") = py::int_(precision); + cls.attr("message_size") = py::int_(itch::encode_message(itch::Message {MsgType {}}).size()); + return cls; +} + +constexpr int STANDARD_PRECISION = 4; +constexpr int MWCB_PRECISION = 8; + +// A factory that default-constructs each message type for `create_message`. +using Factory = std::function; + +} // namespace + +PYBIND11_MODULE(_itchcpp, mod) { + mod.doc() = + "Native (C++) NASDAQ TotalView-ITCH 5.0 backend for the pure-Python `itch` " + "(itchfeed) package: a MessageParser plus typed message classes with matching " + "raw attributes and decode/decode_price/to_bytes semantics."; + + // Maps a message-type byte to its Python class and to a default-constructing + // factory, powering the module-level `messages` dict and `create_message`. + auto type_to_class = std::make_shared(); + auto type_to_factory = std::make_shared>(); + + const auto register_message = [&](char type_byte, const py::object& cls, Factory factory) { + (*type_to_class)[py::bytes(&type_byte, 1)] = cls; + (*type_to_factory)[type_byte] = std::move(factory); + }; + + // Message classes + { + auto cls = bind_common( + mod, "SystemEventMessage", "System Event Message", STANDARD_PRECISION + ); + def_byte(cls, "event_code", &itch::SystemEventMessage::event_code); + register_message('S', cls, [] { return py::cast(itch::SystemEventMessage {}); }); + } + { + auto cls = bind_common( + mod, "StockDirectoryMessage", "Stock Directory Message", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::StockDirectoryMessage::stock); + def_byte(cls, "market_category", &itch::StockDirectoryMessage::market_category); + def_byte( + cls, + "financial_status_indicator", + &itch::StockDirectoryMessage::financial_status_indicator + ); + cls.def_readwrite("round_lot_size", &itch::StockDirectoryMessage::round_lot_size); + def_byte(cls, "round_lots_only", &itch::StockDirectoryMessage::round_lots_only); + def_byte(cls, "issue_classification", &itch::StockDirectoryMessage::issue_classification); + def_byte_array(cls, "issue_sub_type", &itch::StockDirectoryMessage::issue_sub_type); + def_byte(cls, "authenticity", &itch::StockDirectoryMessage::authenticity); + def_byte( + cls, + "short_sale_threshold_indicator", + &itch::StockDirectoryMessage::short_sale_threshold_indicator + ); + def_byte(cls, "ipo_flag", &itch::StockDirectoryMessage::ipo_flag); + def_byte(cls, "luld_ref", &itch::StockDirectoryMessage::luld_ref); + def_byte(cls, "etp_flag", &itch::StockDirectoryMessage::etp_flag); + cls.def_readwrite("etp_leverage_factor", &itch::StockDirectoryMessage::etp_leverage_factor); + def_byte(cls, "inverse_indicator", &itch::StockDirectoryMessage::inverse_indicator); + register_message('R', cls, [] { return py::cast(itch::StockDirectoryMessage {}); }); + } + { + auto cls = bind_common( + mod, "StockTradingActionMessage", "Stock Trading Action Message", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::StockTradingActionMessage::stock); + def_byte(cls, "trading_state", &itch::StockTradingActionMessage::trading_state); + def_byte(cls, "reserved", &itch::StockTradingActionMessage::reserved); + def_byte_array(cls, "reason", &itch::StockTradingActionMessage::reason); + register_message('H', cls, [] { return py::cast(itch::StockTradingActionMessage {}); }); + } + { + auto cls = bind_common( + mod, + "RegSHOMessage", + "Reg SHO Short Sale Price Test Restricted Indicator", + STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::RegSHOMessage::stock); + def_byte(cls, "reg_sho_action", &itch::RegSHOMessage::reg_sho_action); + register_message('Y', cls, [] { return py::cast(itch::RegSHOMessage {}); }); + } + { + auto cls = bind_common( + mod, + "MarketParticipantPositionMessage", + "Market Participant Position message", + STANDARD_PRECISION + ); + def_byte_array(cls, "mpid", &itch::MarketParticipantPositionMessage::mpid); + def_byte_array(cls, "stock", &itch::MarketParticipantPositionMessage::stock); + def_byte( + cls, + "primary_market_maker", + &itch::MarketParticipantPositionMessage::primary_market_maker + ); + def_byte( + cls, "market_maker_mode", &itch::MarketParticipantPositionMessage::market_maker_mode + ); + def_byte( + cls, + "market_participant_state", + &itch::MarketParticipantPositionMessage::market_participant_state + ); + register_message('L', cls, [] { + return py::cast(itch::MarketParticipantPositionMessage {}); + }); + } + { + // Upstream spells this class `MWCBDeclineLeveMessage` (sic) and names the + // levels `levelN_price`; mirror both for drop-in compatibility. + auto cls = bind_common( + mod, + "MWCBDeclineLeveMessage", + "Market wide circuit breaker Decline Level Message", + MWCB_PRECISION + ); + cls.def_readwrite("level1_price", &itch::MWCBDeclineLevelMessage::level1); + cls.def_readwrite("level2_price", &itch::MWCBDeclineLevelMessage::level2); + cls.def_readwrite("level3_price", &itch::MWCBDeclineLevelMessage::level3); + register_message('V', cls, [] { return py::cast(itch::MWCBDeclineLevelMessage {}); }); + } + { + auto cls = bind_common( + mod, + "MWCBStatusMessage", + "Market-Wide Circuit Breaker Status message", + STANDARD_PRECISION + ); + def_byte(cls, "breached_level", &itch::MWCBStatusMessage::breached_level); + register_message('W', cls, [] { return py::cast(itch::MWCBStatusMessage {}); }); + } + { + auto cls = bind_common( + mod, + "IPOQuotingPeriodUpdateMessage", + "IPO Quoting Period Update Message", + STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::IPOQuotingPeriodUpdateMessage::stock); + cls.def_readwrite( + "ipo_release_time", &itch::IPOQuotingPeriodUpdateMessage::ipo_quotation_release_time + ); + def_byte( + cls, + "ipo_release_qualifier", + &itch::IPOQuotingPeriodUpdateMessage::ipo_quotation_release_qualifier + ); + cls.def_readwrite("ipo_price", &itch::IPOQuotingPeriodUpdateMessage::ipo_price); + register_message('K', cls, [] { return py::cast(itch::IPOQuotingPeriodUpdateMessage {}); }); + } + { + auto cls = bind_common( + mod, "LULDAuctionCollarMessage", "LULD Auction Collar", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::LULDAuctionCollarMessage::stock); + cls.def_readwrite( + "auction_collar_reference_price", + &itch::LULDAuctionCollarMessage::auction_collar_reference_price + ); + cls.def_readwrite( + "upper_auction_collar_price", + &itch::LULDAuctionCollarMessage::upper_auction_collar_price + ); + cls.def_readwrite( + "lower_auction_collar_price", + &itch::LULDAuctionCollarMessage::lower_auction_collar_price + ); + // Upstream attribute keeps the original `extention` spelling. + cls.def_readwrite( + "auction_collar_extention", &itch::LULDAuctionCollarMessage::auction_collar_extension + ); + register_message('J', cls, [] { return py::cast(itch::LULDAuctionCollarMessage {}); }); + } + { + auto cls = bind_common( + mod, "OperationalHaltMessage", "Operational Halt", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::OperationalHaltMessage::stock); + def_byte(cls, "market_code", &itch::OperationalHaltMessage::market_code); + def_byte( + cls, "operational_halt_action", &itch::OperationalHaltMessage::operational_halt_action + ); + register_message('h', cls, [] { return py::cast(itch::OperationalHaltMessage {}); }); + } + { + auto cls = bind_common( + mod, + "AddOrderNoMPIAttributionMessage", + "Add Order - No MPID Attribution Message", + STANDARD_PRECISION + ); + cls.def_readwrite("order_reference_number", &itch::AddOrderMessage::order_reference_number); + def_byte(cls, "buy_sell_indicator", &itch::AddOrderMessage::buy_sell_indicator); + cls.def_readwrite("shares", &itch::AddOrderMessage::shares); + def_byte_array(cls, "stock", &itch::AddOrderMessage::stock); + cls.def_readwrite("price", &itch::AddOrderMessage::price); + register_message('A', cls, [] { return py::cast(itch::AddOrderMessage {}); }); + } + { + auto cls = bind_common( + mod, + "AddOrderMPIDAttribution", + "Add Order - MPID Attribution Message", + STANDARD_PRECISION + ); + cls.def_readwrite( + "order_reference_number", &itch::AddOrderMPIDAttributionMessage::order_reference_number + ); + def_byte( + cls, "buy_sell_indicator", &itch::AddOrderMPIDAttributionMessage::buy_sell_indicator + ); + cls.def_readwrite("shares", &itch::AddOrderMPIDAttributionMessage::shares); + def_byte_array(cls, "stock", &itch::AddOrderMPIDAttributionMessage::stock); + cls.def_readwrite("price", &itch::AddOrderMPIDAttributionMessage::price); + def_byte_array(cls, "attribution", &itch::AddOrderMPIDAttributionMessage::attribution); + register_message('F', cls, [] { + return py::cast(itch::AddOrderMPIDAttributionMessage {}); + }); + } + { + auto cls = bind_common( + mod, "OrderExecutedMessage", "Add Order - Order Executed Message", STANDARD_PRECISION + ); + cls.def_readwrite( + "order_reference_number", &itch::OrderExecutedMessage::order_reference_number + ); + cls.def_readwrite("executed_shares", &itch::OrderExecutedMessage::executed_shares); + cls.def_readwrite("match_number", &itch::OrderExecutedMessage::match_number); + register_message('E', cls, [] { return py::cast(itch::OrderExecutedMessage {}); }); + } + { + auto cls = bind_common( + mod, + "OrderExecutedWithPriceMessage", + "Add Order - Order Executed with Price Message", + STANDARD_PRECISION + ); + cls.def_readwrite( + "order_reference_number", &itch::OrderExecutedWithPriceMessage::order_reference_number + ); + cls.def_readwrite("executed_shares", &itch::OrderExecutedWithPriceMessage::executed_shares); + cls.def_readwrite("match_number", &itch::OrderExecutedWithPriceMessage::match_number); + def_byte(cls, "printable", &itch::OrderExecutedWithPriceMessage::printable); + cls.def_readwrite("execution_price", &itch::OrderExecutedWithPriceMessage::execution_price); + register_message('C', cls, [] { return py::cast(itch::OrderExecutedWithPriceMessage {}); }); + } + { + auto cls = bind_common( + mod, "OrderCancelMessage", "Order Cancel Message", STANDARD_PRECISION + ); + cls.def_readwrite( + "order_reference_number", &itch::OrderCancelMessage::order_reference_number + ); + cls.def_readwrite("cancelled_shares", &itch::OrderCancelMessage::cancelled_shares); + register_message('X', cls, [] { return py::cast(itch::OrderCancelMessage {}); }); + } + { + auto cls = bind_common( + mod, "OrderDeleteMessage", "Order Delete Message", STANDARD_PRECISION + ); + cls.def_readwrite( + "order_reference_number", &itch::OrderDeleteMessage::order_reference_number + ); + register_message('D', cls, [] { return py::cast(itch::OrderDeleteMessage {}); }); + } + { + auto cls = bind_common( + mod, "OrderReplaceMessage", "Order Replace Message", STANDARD_PRECISION + ); + cls.def_readwrite( + "order_reference_number", &itch::OrderReplaceMessage::original_order_reference_number + ); + cls.def_readwrite( + "new_order_reference_number", &itch::OrderReplaceMessage::new_order_reference_number + ); + cls.def_readwrite("shares", &itch::OrderReplaceMessage::shares); + cls.def_readwrite("price", &itch::OrderReplaceMessage::price); + register_message('U', cls, [] { return py::cast(itch::OrderReplaceMessage {}); }); + } + { + auto cls = bind_common( + mod, "NonCrossTradeMessage", "Trade Message", STANDARD_PRECISION + ); + cls.def_readwrite( + "order_reference_number", &itch::NonCrossTradeMessage::order_reference_number + ); + def_byte(cls, "buy_sell_indicator", &itch::NonCrossTradeMessage::buy_sell_indicator); + cls.def_readwrite("shares", &itch::NonCrossTradeMessage::shares); + def_byte_array(cls, "stock", &itch::NonCrossTradeMessage::stock); + cls.def_readwrite("price", &itch::NonCrossTradeMessage::price); + cls.def_readwrite("match_number", &itch::NonCrossTradeMessage::match_number); + register_message('P', cls, [] { return py::cast(itch::NonCrossTradeMessage {}); }); + } + { + auto cls = bind_common( + mod, "CrossTradeMessage", "Cross Trade Message", STANDARD_PRECISION + ); + cls.def_readwrite("shares", &itch::CrossTradeMessage::shares); + def_byte_array(cls, "stock", &itch::CrossTradeMessage::stock); + cls.def_readwrite("cross_price", &itch::CrossTradeMessage::cross_price); + cls.def_readwrite("match_number", &itch::CrossTradeMessage::match_number); + def_byte(cls, "cross_type", &itch::CrossTradeMessage::cross_type); + register_message('Q', cls, [] { return py::cast(itch::CrossTradeMessage {}); }); + } + { + auto cls = bind_common( + mod, "BrokenTradeMessage", "Broken Trade Message", STANDARD_PRECISION + ); + cls.def_readwrite("match_number", &itch::BrokenTradeMessage::match_number); + register_message('B', cls, [] { return py::cast(itch::BrokenTradeMessage {}); }); + } + { + auto cls = + bind_common(mod, "NOIIMessage", "NOII Message", STANDARD_PRECISION); + cls.def_readwrite("paired_shares", &itch::NOIIMessage::paired_shares); + cls.def_readwrite("imbalance_shares", &itch::NOIIMessage::imbalance_shares); + def_byte(cls, "imbalance_direction", &itch::NOIIMessage::imbalance_direction); + def_byte_array(cls, "stock", &itch::NOIIMessage::stock); + cls.def_readwrite("far_price", &itch::NOIIMessage::far_price); + cls.def_readwrite("near_price", &itch::NOIIMessage::near_price); + cls.def_readwrite("current_reference_price", &itch::NOIIMessage::current_reference_price); + def_byte(cls, "cross_type", &itch::NOIIMessage::cross_type); + def_byte(cls, "variation_indicator", &itch::NOIIMessage::price_variation_indicator); + register_message('I', cls, [] { return py::cast(itch::NOIIMessage {}); }); + } + { + auto cls = bind_common( + mod, "RetailPriceImprovementIndicator", "Retail Interest message", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::RetailPriceImprovementIndicatorMessage::stock); + def_byte( + cls, "interest_flag", &itch::RetailPriceImprovementIndicatorMessage::interest_flag + ); + register_message('N', cls, [] { + return py::cast(itch::RetailPriceImprovementIndicatorMessage {}); + }); + } + { + auto cls = bind_common( + mod, "DLCRMessage", "Direct Listing with Capital Raise Message", STANDARD_PRECISION + ); + def_byte_array(cls, "stock", &itch::DLCRMessage::stock); + def_byte(cls, "open_eligibility_status", &itch::DLCRMessage::open_eligibility_status); + cls.def_readwrite("minimum_allowable_price", &itch::DLCRMessage::minimum_allowable_price); + cls.def_readwrite("maximum_allowable_price", &itch::DLCRMessage::maximum_allowable_price); + cls.def_readwrite("near_execution_price", &itch::DLCRMessage::near_execution_price); + cls.def_readwrite("near_execution_time", &itch::DLCRMessage::near_execution_time); + cls.def_readwrite("lower_price_range_collar", &itch::DLCRMessage::lower_price_range_collar); + cls.def_readwrite("upper_price_range_collar", &itch::DLCRMessage::upper_price_range_collar); + register_message('O', cls, [] { return py::cast(itch::DLCRMessage {}); }); + } + + // Module-level message registry and factory + mod.attr("AllMessages") = py::bytes(std::string {ALL_MESSAGES}); + mod.attr("messages") = *type_to_class; + + mod.def( + "create_message", + [type_to_factory](const py::bytes& message_type, const py::kwargs& kwargs) -> py::object { + const std::string type_text {message_type}; + if (type_text.size() != 1) { + throw py::value_error("message_type must be a single byte"); + } + const auto factory = type_to_factory->find(type_text.front()); + if (factory == type_to_factory->end()) { + throw py::value_error("Unknown message type: " + type_text); + } + py::object instance = factory->second(); + constexpr std::uint64_t timestamp_mask {(std::uint64_t {1} << 48U) - 1U}; + for (const auto& item : kwargs) { + if (item.first.cast() == "timestamp") { + instance.attr("timestamp") = + py::int_(item.second.cast() & timestamp_mask); + } else { + instance.attr(item.first) = item.second; + } + } + return instance; + }, + py::arg("message_type"), + "Create a message of the given type byte, populated from keyword attributes." + ); + + // Parser + // A lazy iterator over framed ITCH messages, applying the type filter, the + // optional save_file re-emission, and the stop-on-system-event-'C' rule. When + // `m_file` is a binary file object the buffer is refilled `m_cachesize` bytes at + // a time (consumed bytes are dropped), so a whole-day feed never has to be held + // in memory at once; for an in-memory stream `m_file` is None. + struct MessageIterator { + std::string m_buffer; + std::size_t m_offset {0}; + std::string m_filter; + py::object m_file; + std::size_t m_cachesize {DEFAULT_CACHE_SIZE}; + py::object m_save_file; + bool m_stopped {false}; + }; + + py::class_(mod, "_MessageIterator") + .def("__iter__", [](py::object self) { return self; }) + .def("__next__", [](MessageIterator& iter) -> py::object { + // Ensures at least `need` unconsumed bytes are buffered, pulling more + // from the file (and dropping already-consumed bytes) when possible. + const auto fill_to = [&iter](std::size_t need) -> bool { + while (iter.m_buffer.size() - iter.m_offset < need) { + if (iter.m_file.is_none()) { + return false; + } + if (iter.m_offset > 0) { + iter.m_buffer.erase(0, iter.m_offset); + iter.m_offset = 0; + } + const std::string chunk { + iter.m_file.attr("read")(iter.m_cachesize).cast() + }; + if (chunk.empty()) { + return false; + } + iter.m_buffer += chunk; + } + return true; + }; + + while (true) { + if (iter.m_stopped || !fill_to(FRAME_HEADER_LEN)) { + throw py::stop_iteration(); + } + const auto high = static_cast(iter.m_buffer[iter.m_offset]); + const auto low = static_cast(iter.m_buffer[iter.m_offset + 1]); + const std::size_t body_len {(static_cast(high) << 8U) | low}; + const std::size_t total {FRAME_HEADER_LEN + body_len}; + // fill_to may compact the buffer (resetting m_offset), so read the + // frame position only after it returns. + if (!fill_to(total)) { + throw py::stop_iteration(); // Incomplete trailing message. + } + const std::size_t pos = iter.m_offset; + + const char type_byte = iter.m_buffer[pos + FRAME_HEADER_LEN]; + if (ALL_MESSAGES.find(type_byte) == std::string_view::npos) { + throw py::value_error(std::string {"Unknown message type: "} + type_byte); + } + + itch::Parser parser; + std::optional parsed; + parser.parse( + iter.m_buffer.data() + pos, total, [&parsed](const itch::Message& message) { + parsed = message; + } + ); + const std::string_view frame {iter.m_buffer.data() + pos, total}; + iter.m_offset = pos + total; + + const bool kept = iter.m_filter.find(type_byte) != std::string::npos; + const bool is_stop = type_byte == 'S' && + std::get(*parsed).event_code == 'C'; + + if (kept) { + if (!iter.m_save_file.is_none()) { + iter.m_save_file.attr("write")(py::bytes(frame.data(), frame.size())); + } + if (is_stop) { + iter.m_stopped = true; + } + return message_to_object(*parsed); + } + if (is_stop) { + throw py::stop_iteration(); + } + } + }); + + struct MessageParser { + std::string m_filter; + }; + + const auto make_iterator = [](const std::string& filter, + std::string buffer, + py::object file, + std::size_t cachesize, + const py::object& save_file) { + MessageIterator iter; + iter.m_buffer = std::move(buffer); + iter.m_filter = filter; + iter.m_file = std::move(file); + iter.m_cachesize = cachesize; + iter.m_save_file = save_file; + return iter; + }; + + py::class_(mod, "MessageParser") + .def( + py::init([](const py::bytes& message_type) { + return MessageParser {std::string {message_type}}; + }), + py::arg("message_type") = py::bytes(std::string {ALL_MESSAGES}) + ) + .def_property_readonly( + "message_type", [](const MessageParser& self) { return py::bytes(self.m_filter); } + ) + .def( + "get_message_type", + [](const MessageParser&, const py::bytes& message) { + const std::string body {message}; + if (body.empty() || ALL_MESSAGES.find(body.front()) == std::string_view::npos) { + throw py::value_error( + std::string {"Unknown message type: "} + (body.empty() ? '?' : body.front()) + ); + } + return message_to_object(unpack_body(body)); + }, + py::arg("message") + ) + .def( + "parse_stream", + [make_iterator]( + const MessageParser& self, const py::bytes& data, const py::object& save_file + ) { + return make_iterator( + self.m_filter, std::string {data}, py::none(), DEFAULT_CACHE_SIZE, save_file + ); + }, + py::arg("data"), + py::arg("save_file") = py::none() + ) + .def( + "parse_file", + [make_iterator]( + const MessageParser& self, + const py::object& file, + std::size_t cachesize, + const py::object& save_file + ) { + // The file is read lazily, `cachesize` bytes at a time, by the iterator. + return make_iterator(self.m_filter, std::string {}, file, cachesize, save_file); + }, + py::arg("file"), + py::arg("cachesize") = DEFAULT_CACHE_SIZE, + py::arg("save_file") = py::none() + ) + .def( + "parse_messages", + [make_iterator]( + const MessageParser& self, const py::object& data, const py::object& callback + ) { + const bool is_buffer = + py::isinstance(data) || py::isinstance(data); + MessageIterator iter = + is_buffer + ? make_iterator( + self.m_filter, + std::string {py::bytes(data)}, + py::none(), + DEFAULT_CACHE_SIZE, + py::none() + ) + : make_iterator( + self.m_filter, std::string {}, data, DEFAULT_CACHE_SIZE, py::none() + ); + const py::object next = py::cast(iter).attr("__next__"); + while (true) { + py::object message; + try { + message = next(); + } catch (py::error_already_set& stop) { + if (stop.matches(PyExc_StopIteration)) { + break; + } + throw; + } + callback(message); + } + }, + py::arg("data"), + py::arg("callback") + ); + + // Book engine (native extras, not part of upstream `itch`) + py::class_(mod, "Bbo") + .def_readonly("has_bid", &itch::book::Bbo::has_bid) + .def_readonly("has_ask", &itch::book::Bbo::has_ask) + .def_readonly("bid_shares", &itch::book::Bbo::bid_shares) + .def_readonly("ask_shares", &itch::book::Bbo::ask_shares) + .def_property_readonly( + "bid_price", [](const itch::book::Bbo& bbo) { return bbo.bid_price.to_double(); } + ) + .def_property_readonly("ask_price", [](const itch::book::Bbo& bbo) { + return bbo.ask_price.to_double(); + }); + + py::class_(mod, "L3Book") + .def_property_readonly("symbol", &itch::book::L3Book::symbol) + .def("bbo", &itch::book::L3Book::bbo); + + py::class_(mod, "BookManager") + .def(py::init<>()) + .def("track_symbol", &itch::book::BookManager::track_symbol, py::arg("symbol")) + .def( + "process_stream", + [](itch::book::BookManager& manager, const py::bytes& data) { + const std::string buffer {data}; + itch::Parser parser; + parser.parse(buffer.data(), buffer.size(), [&manager](const itch::Message& msg) { + manager.process(msg); + }); + }, + py::arg("raw_binary_data"), + "Parse a raw ITCH buffer and update every symbol's book in one pass." + ) + .def("book_count", &itch::book::BookManager::book_count) + .def( + "book_for_symbol", + &itch::book::BookManager::book_for_symbol, + py::arg("symbol"), + py::return_value_policy::reference_internal + ); + + // Analytics (native extra) + py::class_(mod, "Vwap") + .def(py::init<>()) + .def( + "add", + [](itch::analytics::Vwap& vwap, std::uint32_t raw_price, std::uint64_t shares) { + vwap.add(itch::StandardPrice {raw_price}, shares); + }, + py::arg("raw_price"), + py::arg("shares") + ) + .def("value", &itch::analytics::Vwap::value) + .def("volume", &itch::analytics::Vwap::volume) + .def("reset", &itch::analytics::Vwap::reset); +} diff --git a/python/tests/data.py b/python/tests/data.py new file mode 100644 index 0000000..ea61eb6 --- /dev/null +++ b/python/tests/data.py @@ -0,0 +1,230 @@ +"""Sample message payloads for every ITCH 5.0 message type. + +Vendored from the upstream ``itchfeed`` test suite so the native backend can be +exercised against the same fixtures. +""" + +DEFAULT_STOCK_LOCATE = 1 +DEFAULT_TRACKING_NUMBER = 2 +DEFAULT_TIMESTAMP = 1651500000 * 1_000_000_000 +DEFAULT_ORDER_REF = 1234567890 +DEFAULT_MATCH_NUM = 9876543210 +DEFAULT_STOCK = b"AAPL " +DEFAULT_MPID = b"NSDQ" + +TEST_DATA = { + b"S": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "event_code": b"O", + }, + b"R": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "market_category": b"Q", + "financial_status_indicator": b"N", + "round_lot_size": 100, + "round_lots_only": b"N", + "issue_classification": b"C", + "issue_sub_type": b"I ", + "authenticity": b"P", + "short_sale_threshold_indicator": b"N", + "ipo_flag": b"N", + "luld_ref": b"1", + "etp_flag": b"N", + "etp_leverage_factor": 0, + "inverse_indicator": b"N", + }, + b"H": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "trading_state": b"T", + "reserved": b"\x00", + "reason": b"T1 ", + }, + b"Y": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "reg_sho_action": b"0", + }, + b"L": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "mpid": DEFAULT_MPID, + "stock": DEFAULT_STOCK, + "primary_market_maker": b"Y", + "market_maker_mode": b"N", + "market_participant_state": b"A", + }, + b"V": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "level1_price": 10000000000, + "level2_price": 20000000000, + "level3_price": 30000000000, + }, + b"W": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "breached_level": b"1", + }, + b"K": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "ipo_release_time": 34200, + "ipo_release_qualifier": b"A", + "ipo_price": 255000, + }, + b"J": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "auction_collar_reference_price": 1500000, + "upper_auction_collar_price": 1600000, + "lower_auction_collar_price": 1400000, + "auction_collar_extention": 1, + }, + b"h": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "market_code": b"Q", + "operational_halt_action": b"H", + }, + b"A": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "buy_sell_indicator": b"B", + "shares": 100, + "stock": DEFAULT_STOCK, + "price": 1501234, + }, + b"F": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "buy_sell_indicator": b"S", + "shares": 200, + "stock": DEFAULT_STOCK, + "price": 1505000, + "attribution": DEFAULT_MPID, + }, + b"E": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "executed_shares": 50, + "match_number": DEFAULT_MATCH_NUM, + }, + b"C": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "executed_shares": 50, + "match_number": DEFAULT_MATCH_NUM, + "printable": b"Y", + "execution_price": 1502000, + }, + b"X": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "cancelled_shares": 100, + }, + b"D": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + }, + b"U": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "new_order_reference_number": DEFAULT_ORDER_REF + 1, + "shares": 150, + "price": 1510000, + }, + b"P": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "order_reference_number": DEFAULT_ORDER_REF, + "buy_sell_indicator": b"B", + "shares": 100, + "stock": DEFAULT_STOCK, + "price": 1503000, + "match_number": DEFAULT_MATCH_NUM, + }, + b"Q": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "shares": 100000, + "stock": DEFAULT_STOCK, + "cross_price": 1500000, + "match_number": DEFAULT_MATCH_NUM, + "cross_type": b"O", + }, + b"B": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "match_number": DEFAULT_MATCH_NUM, + }, + b"I": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "paired_shares": 50000, + "imbalance_shares": 10000, + "imbalance_direction": b"B", + "stock": DEFAULT_STOCK, + "far_price": 1520000, + "near_price": 1510000, + "current_reference_price": 1500000, + "cross_type": b"O", + "variation_indicator": b" ", + }, + b"N": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "interest_flag": b"A", + }, + b"O": { + "stock_locate": DEFAULT_STOCK_LOCATE, + "tracking_number": DEFAULT_TRACKING_NUMBER, + "timestamp": DEFAULT_TIMESTAMP, + "stock": DEFAULT_STOCK, + "open_eligibility_status": b"Y", + "minimum_allowable_price": 1800000, + "maximum_allowable_price": 2200000, + "near_execution_price": 2000000, + "near_execution_time": DEFAULT_TIMESTAMP, + "lower_price_range_collar": 1900000, + "upper_price_range_collar": 2100000, + }, +} diff --git a/python/tests/test_bindings.py b/python/tests/test_bindings.py new file mode 100644 index 0000000..a90f791 --- /dev/null +++ b/python/tests/test_bindings.py @@ -0,0 +1,71 @@ +"""Smoke tests for the native ``itchcpp`` bindings. + +Covers the top-level package surface (parser, raw attribute semantics) and the +native extras (the order-book engine and the VWAP accumulator) that go beyond the +pure-Python ``itch`` API. +""" + +import struct + +import itchcpp +from itchcpp.parser import MessageParser + + +def _frame(payload: bytes) -> bytes: + return struct.pack(">H", len(payload)) + payload + + +def _add_order( + locate: int, ref: int, side: bytes, shares: int, sym: bytes, price: int +) -> bytes: + payload = ( + b"A" + + struct.pack(">HH", locate, 0) + + (1234).to_bytes(6, "big") + + struct.pack(">Q", ref) + + side + + struct.pack(">I", shares) + + sym.ljust(8) + + struct.pack(">I", price) + ) + return _frame(payload) + + +def test_parse_stream_yields_typed_messages_with_raw_attributes() -> None: + buffer = _add_order(7, 42, b"B", 500, b"AAPL", 1_500_000) + messages = list(MessageParser().parse_stream(buffer)) + + assert len(messages) == 1 + message = messages[0] + assert isinstance(message, itchcpp.AddOrderNoMPIAttributionMessage) + # Raw attributes are bytes/int, exactly like the pure-Python package. + assert message.message_type == b"A" + assert message.stock == b"AAPL " + assert message.shares == 500 + assert message.order_reference_number == 42 + assert message.price == 1_500_000 + # decode_price applies the 4-decimal scale; decode() trims the symbol. + assert message.decode_price("price") == 150.0 + assert message.decode().stock == "AAPL" + + +def test_book_manager_reconstructs_bbo() -> None: + buffer = _add_order(7, 1, b"B", 300, b"AAPL", 1_500_000) + _add_order( + 7, 2, b"S", 100, b"AAPL", 1_500_200 + ) + manager = itchcpp.book.BookManager() + manager.process_stream(buffer) + + assert manager.book_count() == 1 + bbo = manager.book_for_symbol("AAPL").bbo() + assert bbo.bid_price == 150.0 + assert bbo.ask_price == 150.02 + assert bbo.bid_shares == 300 + + +def test_vwap_accumulates() -> None: + vwap = itchcpp.analytics.Vwap() + vwap.add(100, 10) + vwap.add(200, 30) + assert vwap.volume() == 40 + assert abs(vwap.value() - (175.0 / 10000.0)) < 1e-12 diff --git a/python/tests/test_parity.py b/python/tests/test_parity.py new file mode 100644 index 0000000..5585943 --- /dev/null +++ b/python/tests/test_parity.py @@ -0,0 +1,206 @@ +"""Behavioral parity tests for the native ``itchcpp`` backend. + +These mirror the upstream ``itchfeed`` test suite, exercised through the public +API (``create_message`` -> ``to_bytes`` -> frame -> ``parse_stream``/``parse_file``, +plus ``decode`` / ``decode_price`` / ``get_attributes``) so the native module is +verified to behave as a drop-in replacement. +""" + +from __future__ import annotations + +from dataclasses import is_dataclass +from io import BytesIO + +import pytest + +import itchcpp.messages as msgs +from data import TEST_DATA +from itchcpp.messages import create_message, messages +from itchcpp.parser import MessageParser + + +def _frame(body: bytes) -> bytes: + """Wrap a message body in the ITCH framing the parser expects.""" + return b"\x00" + len(body).to_bytes(1, "big") + body + + +# Parser behavior + + +def test_parse_stream_single_message() -> None: + body = create_message(b"S", **TEST_DATA[b"S"]).to_bytes() + messages_out = list(MessageParser().parse_stream(_frame(body))) + assert len(messages_out) == 1 + assert isinstance(messages_out[0], msgs.SystemEventMessage) + + +def test_parse_stream_multiple_messages() -> None: + data = _frame(create_message(b"S", **TEST_DATA[b"S"]).to_bytes()) + _frame( + create_message(b"A", **TEST_DATA[b"A"]).to_bytes() + ) + out = list(MessageParser().parse_stream(data)) + assert len(out) == 2 + assert isinstance(out[0], msgs.SystemEventMessage) + assert isinstance(out[1], msgs.AddOrderNoMPIAttributionMessage) + + +def test_parse_file_reads_binary_object() -> None: + body = create_message(b"S", **TEST_DATA[b"S"]).to_bytes() + out = list(MessageParser().parse_file(BytesIO(_frame(body)))) + assert len(out) == 1 + assert isinstance(out[0], msgs.SystemEventMessage) + + +def test_parse_file_streams_in_chunks() -> None: + # Many messages with a tiny cache size forces the iterator to refill its buffer + # across frame boundaries, exercising the lazy chunked-read path. + one = _frame(create_message(b"A", **TEST_DATA[b"A"]).to_bytes()) + blob = one * 50 + out = list(MessageParser().parse_file(BytesIO(blob), cachesize=7)) + assert len(out) == 50 + assert all(isinstance(m, msgs.AddOrderNoMPIAttributionMessage) for m in out) + assert out[0].order_reference_number == TEST_DATA[b"A"]["order_reference_number"] + + +def test_unknown_message_type_raises() -> None: + with pytest.raises(ValueError): + list(MessageParser().parse_stream(b"\x00\x01\x00")) + + +def test_incomplete_message_is_skipped() -> None: + body = create_message(b"S", **TEST_DATA[b"S"]).to_bytes() + data = _frame(body)[:-1] + assert list(MessageParser().parse_stream(data)) == [] + + +def test_stop_on_system_event_c() -> None: + kwargs = dict(TEST_DATA[b"S"], event_code=b"C") + data = _frame(create_message(b"S", **kwargs).to_bytes()) + _frame( + create_message(b"A", **TEST_DATA[b"A"]).to_bytes() + ) + out = list(MessageParser().parse_stream(data)) + assert len(out) == 1 + assert isinstance(out[0], msgs.SystemEventMessage) + + +def test_message_type_filter() -> None: + data = _frame(create_message(b"S", **TEST_DATA[b"S"]).to_bytes()) + _frame( + create_message(b"A", **TEST_DATA[b"A"]).to_bytes() + ) + out = list(MessageParser(message_type=b"A").parse_stream(data)) + assert len(out) == 1 + assert isinstance(out[0], msgs.AddOrderNoMPIAttributionMessage) + + +def test_parse_messages_callback() -> None: + data = _frame(create_message(b"S", **TEST_DATA[b"S"]).to_bytes()) + collected: list = [] + MessageParser().parse_messages(data, collected.append) + assert len(collected) == 1 + assert isinstance(collected[0], msgs.SystemEventMessage) + + +@pytest.mark.parametrize("message_type", list(TEST_DATA.keys())) +def test_parse_all_message_types(message_type: bytes) -> None: + body = create_message(message_type, **TEST_DATA[message_type]).to_bytes() + out = list(MessageParser().parse_stream(_frame(body))) + assert len(out) == 1 + assert isinstance(out[0], messages[message_type]) + + +# create_message / pack / unpack / decode + + +@pytest.mark.parametrize("message_type", list(TEST_DATA.keys())) +def test_create_message_round_trips(message_type: bytes) -> None: + sample = TEST_DATA[message_type] + body = create_message(message_type, **sample).to_bytes() + + assert body.startswith(message_type) + assert len(body) == messages[message_type].message_size + + unpacked = messages[message_type](body) + for key, expected in sample.items(): + if key == "timestamp": + expected &= (1 << 48) - 1 + assert getattr(unpacked, key) == expected, f"{message_type!r}.{key}" + + assert unpacked.to_bytes() == body + + +@pytest.mark.parametrize("message_type", list(TEST_DATA.keys())) +def test_decode_produces_prefixed_dataclass(message_type: bytes) -> None: + sample = TEST_DATA[message_type] + instance = messages[message_type](create_message(message_type, **sample).to_bytes()) + + decoded = instance.decode(prefix="Test") + assert is_dataclass(decoded) + assert decoded.__class__.__name__ == f"Test{messages[message_type].__name__}" + + for key, expected in sample.items(): + decoded_value = getattr(decoded, key) + if key == "timestamp": + assert decoded_value == (expected & ((1 << 48) - 1)) + elif "price" in key and key not in ("price_precision",): + precision = 8 if key.startswith("level") else 4 + assert decoded_value == pytest.approx(expected / (10**precision)) + elif isinstance(expected, bytes): + assert decoded_value == expected.decode("ascii").rstrip() + else: + assert decoded_value == expected + + +def test_get_attributes_partitions_data_and_methods() -> None: + sample = TEST_DATA[b"A"] + instance = messages[b"A"](create_message(b"A", **sample).to_bytes()) + + non_callable = instance.get_attributes(call_able=False) + callable_attrs = instance.get_attributes(call_able=True) + + assert "message_type" in non_callable + assert "description" in non_callable + assert "timestamp" in non_callable + assert "stock" in non_callable + assert "price" in non_callable + + for name in ( + "to_bytes", + "decode", + "decode_price", + "set_timestamp", + "split_timestamp", + ): + assert name in callable_attrs + + assert not set(non_callable) & set(callable_attrs) + for key in sample: + assert key in non_callable + + +def test_decode_price_scales_and_validates() -> None: + add = messages[b"A"](create_message(b"A", **TEST_DATA[b"A"]).to_bytes()) + assert add.decode_price("price") == pytest.approx(TEST_DATA[b"A"]["price"] / 10000) + + mwcb = messages[b"V"](create_message(b"V", **TEST_DATA[b"V"]).to_bytes()) + assert mwcb.decode_price("level1_price") == pytest.approx( + TEST_DATA[b"V"]["level1_price"] / 10**8 + ) + + with pytest.raises( + ValueError, match="Please check the price attribute for to_bytes" + ): + add.decode_price("to_bytes") + with pytest.raises(TypeError): + add.decode_price("stock") + + +def test_isinstance_hierarchy() -> None: + add = messages[b"A"](create_message(b"A", **TEST_DATA[b"A"]).to_bytes()) + trade = messages[b"P"](create_message(b"P", **TEST_DATA[b"P"]).to_bytes()) + execed = messages[b"E"](create_message(b"E", **TEST_DATA[b"E"]).to_bytes()) + + assert isinstance(add, msgs.AddOrderMessage) + assert isinstance(add, msgs.MarketMessage) + assert isinstance(trade, msgs.TradeMessage) + assert isinstance(execed, msgs.ModifyOrderMessage) + assert not isinstance(add, msgs.TradeMessage) diff --git a/scripts/fetch_sample.sh b/scripts/fetch_sample.sh new file mode 100644 index 0000000..ead1da8 --- /dev/null +++ b/scripts/fetch_sample.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Downloads an official NASDAQ TotalView-ITCH 5.0 sample file for use by the +# benchmarks and the golden/conformance tests. The raw sample is a multi- +# gigabyte binary and is deliberately NOT committed to the repository: fetch it +# on demand instead. +# +# Usage: +# scripts/fetch_sample.sh [destination_dir] +# +# Environment: +# ITCH_SAMPLE_URL Override the download URL (defaults to the NASDAQ archive). +# +# The default URL points at NASDAQ's public sample archive. NASDAQ occasionally +# changes the available dates; if the default 404s, browse the index and set +# ITCH_SAMPLE_URL to a current file: +# https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/ + +set -euo pipefail + +readonly DEFAULT_URL="https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/01302020.NASDAQ_ITCH50.gz" +readonly SAMPLE_URL="${ITCH_SAMPLE_URL:-${DEFAULT_URL}}" + +dest_dir="${1:-data}" +mkdir -p "${dest_dir}" + +archive_name="$(basename "${SAMPLE_URL}")" +archive_path="${dest_dir}/${archive_name}" +decompressed_path="${archive_path%.gz}" + +if [[ -f "${decompressed_path}" ]]; then + echo "Sample already present: ${decompressed_path}" + exit 0 +fi + +echo "Downloading ITCH sample from: ${SAMPLE_URL}" +if command -v curl >/dev/null 2>&1; then + curl -fL --retry 3 -o "${archive_path}" "${SAMPLE_URL}" +elif command -v wget >/dev/null 2>&1; then + wget -O "${archive_path}" "${SAMPLE_URL}" +else + echo "error: neither curl nor wget is available" >&2 + exit 1 +fi + +if [[ "${archive_path}" == *.gz ]]; then + echo "Decompressing ${archive_path}" + gzip -d -k "${archive_path}" +fi + +echo "Sample ready: ${decompressed_path}" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fe22a2b..b74fcff 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,14 +1,34 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) include(GNUInstallDirs) # Step 1: Add targets and properties add_library( - itch - parser.cpp - messages.cpp + itch + parser.cpp + messages.cpp order_book.cpp + transport/moldudp64.cpp + transport/soupbintcp.cpp + transport/pcap.cpp + book/l3_book.cpp + book/book_manager.cpp + io/csv_sink.cpp + io/arrow_export.cpp + encoder.cpp + replay.cpp ) + +# Apache Arrow / Parquet columnar export is optional and off by default so the +# core library stays dependency-free. The arrow_export.cpp translation unit is +# empty unless ITCH_WITH_ARROW is defined. +if(${PROJECT_NAME}_WITH_ARROW) + find_package(Arrow CONFIG REQUIRED) + find_package(Parquet CONFIG REQUIRED) + target_compile_definitions(itch PUBLIC ITCH_WITH_ARROW) + target_link_libraries(itch PUBLIC Arrow::arrow_shared Parquet::parquet_shared) + message(STATUS "ITCH: Apache Arrow / Parquet export enabled") +endif() add_library(itch::itch ALIAS itch) if (NOT ANDROID) target_link_libraries( @@ -19,6 +39,11 @@ endif() configure_visibility(itch) +# itch is a static library by default, but the pybind11 module (and any other +# shared-library consumer) links its object code into a .so, which requires +# position-independent code regardless of how itch itself is built. +set_target_properties(itch PROPERTIES POSITION_INDEPENDENT_CODE ON) + # Step 2: Update Include Directories target_include_directories( itch PUBLIC diff --git a/src/book/book_manager.cpp b/src/book/book_manager.cpp new file mode 100644 index 0000000..f67335c --- /dev/null +++ b/src/book/book_manager.cpp @@ -0,0 +1,236 @@ +#include "itch/book/book_manager.hpp" + +#include +#include + +namespace itch::book { + +namespace { + +// Maps an ITCH buy/sell indicator byte to the book Side enum. +auto to_side(char buy_sell_indicator) noexcept -> Side { + return buy_sell_indicator == 'B' ? Side::buy : Side::sell; +} + +} // namespace + +auto BookManager::in_universe(std::string_view symbol) const -> bool { + return m_universe.empty() || m_universe.contains(std::string {symbol}); +} + +auto BookManager::entry(std::uint16_t stock_locate) const -> BookEntry* { + if (stock_locate >= m_books_by_locate.size()) { + return nullptr; + } + return m_books_by_locate[stock_locate].get(); +} + +auto BookManager::ensure_entry(std::uint16_t stock_locate, std::string_view symbol) -> BookEntry* { + if (stock_locate >= m_books_by_locate.size()) { + m_books_by_locate.resize(static_cast(stock_locate) + 1); + } + if (auto& slot = m_books_by_locate[stock_locate]) { + if (slot->book.symbol().empty() && !symbol.empty()) { + slot->book.set_symbol(std::string {symbol}); + } + return slot.get(); + } + if (!in_universe(symbol)) { + return nullptr; + } + m_books_by_locate[stock_locate] = std::make_unique(); + m_books_by_locate[stock_locate]->book.set_symbol(std::string {symbol}); + ++m_book_count; + return m_books_by_locate[stock_locate].get(); +} + +auto BookManager::emit_bbo_if_changed(BookEntry& target) -> void { + const Bbo current = target.book.bbo(); + if (current != target.last_bbo) { + target.last_bbo = current; + if (m_bbo_callback) { + m_bbo_callback(target.book, current); + } + } +} + +template +auto BookManager::handle_add_order(const AddMessage& add) -> void { + BookEntry* target = ensure_entry(add.stock_locate, to_string(add.stock, STOCK_LEN)); + if (target != nullptr) { + target->book.add_order( + add.order_reference_number, to_side(add.buy_sell_indicator), add.shares, add.price + ); + emit_bbo_if_changed(*target); + } +} + +auto BookManager::handle_order_executed(const OrderExecutedMessage& exec) -> void { + BookEntry* target = entry(exec.stock_locate); + if (target == nullptr) { + return; + } + if (m_trade_callback) { + const auto price = target->book.order_price(exec.order_reference_number); + const auto side = target->book.order_side(exec.order_reference_number); + if (price.has_value()) { + Trade trade {}; + trade.timestamp = exec.timestamp; + trade.stock_locate = exec.stock_locate; + trade.symbol = target->book.symbol(); + trade.price = StandardPrice {*price}; + trade.shares = exec.executed_shares; + trade.match_number = exec.match_number; + trade.side = side.has_value() ? static_cast(*side) : '\0'; + trade.printable = true; + m_trade_callback(trade); + } + } + target->book.execute_order(exec.order_reference_number, exec.executed_shares); + emit_bbo_if_changed(*target); +} + +auto BookManager::handle_order_executed_with_price(const OrderExecutedWithPriceMessage& exec +) -> void { + BookEntry* target = entry(exec.stock_locate); + if (target == nullptr) { + return; + } + if (m_trade_callback) { + const auto side = target->book.order_side(exec.order_reference_number); + Trade trade {}; + trade.timestamp = exec.timestamp; + trade.stock_locate = exec.stock_locate; + trade.symbol = target->book.symbol(); + trade.price = StandardPrice {exec.execution_price}; + trade.shares = exec.executed_shares; + trade.match_number = exec.match_number; + trade.side = side.has_value() ? static_cast(*side) : '\0'; + trade.printable = exec.printable == 'Y'; + m_trade_callback(trade); + } + target->book.execute_order(exec.order_reference_number, exec.executed_shares); + emit_bbo_if_changed(*target); +} + +auto BookManager::handle_order_cancel(const OrderCancelMessage& cancel) -> void { + if (BookEntry* target = entry(cancel.stock_locate)) { + target->book.reduce_order(cancel.order_reference_number, cancel.cancelled_shares); + emit_bbo_if_changed(*target); + } +} + +auto BookManager::handle_order_delete(const OrderDeleteMessage& del) -> void { + if (BookEntry* target = entry(del.stock_locate)) { + target->book.delete_order(del.order_reference_number); + emit_bbo_if_changed(*target); + } +} + +auto BookManager::handle_order_replace(const OrderReplaceMessage& replace) -> void { + if (BookEntry* target = entry(replace.stock_locate)) { + target->book.replace_order( + replace.original_order_reference_number, + replace.new_order_reference_number, + replace.shares, + replace.price + ); + emit_bbo_if_changed(*target); + } +} + +auto BookManager::handle_non_cross_trade(const NonCrossTradeMessage& trade) -> void { + // A non-displayable order print does not alter the visible book. + if (!m_trade_callback) { + return; + } + Trade tape_entry {}; + tape_entry.timestamp = trade.timestamp; + tape_entry.stock_locate = trade.stock_locate; + tape_entry.symbol = to_string(trade.stock, STOCK_LEN); + tape_entry.price = StandardPrice {trade.price}; + tape_entry.shares = trade.shares; + tape_entry.match_number = trade.match_number; + tape_entry.side = trade.buy_sell_indicator; + tape_entry.printable = true; + m_trade_callback(tape_entry); +} + +auto BookManager::handle_cross_trade(const CrossTradeMessage& cross) -> void { + if (!m_trade_callback) { + return; + } + Trade tape_entry {}; + tape_entry.timestamp = cross.timestamp; + tape_entry.stock_locate = cross.stock_locate; + tape_entry.symbol = to_string(cross.stock, STOCK_LEN); + tape_entry.price = StandardPrice {cross.cross_price}; + tape_entry.shares = cross.shares; + tape_entry.match_number = cross.match_number; + tape_entry.printable = true; + tape_entry.is_cross = true; + tape_entry.cross_type = cross.cross_type; + m_trade_callback(tape_entry); +} + +auto BookManager::handle_stock_directory(const StockDirectoryMessage& directory) -> void { + if (directory.stock_locate >= m_symbol_by_locate.size()) { + m_symbol_by_locate.resize(static_cast(directory.stock_locate) + 1); + } + m_symbol_by_locate[directory.stock_locate] = to_string(directory.stock, STOCK_LEN); +} + +auto BookManager::process(const Message& message) -> void { + if (const auto* add = std::get_if(&message)) { + handle_add_order(*add); + } else if (const auto* add_mpid = std::get_if(&message)) { + handle_add_order(*add_mpid); + } else if (const auto* executed = std::get_if(&message)) { + handle_order_executed(*executed); + } else if (const auto* executed_with_price = + std::get_if(&message)) { + handle_order_executed_with_price(*executed_with_price); + } else if (const auto* cancel = std::get_if(&message)) { + handle_order_cancel(*cancel); + } else if (const auto* del = std::get_if(&message)) { + handle_order_delete(*del); + } else if (const auto* replace = std::get_if(&message)) { + handle_order_replace(*replace); + } else if (const auto* trade = std::get_if(&message)) { + handle_non_cross_trade(*trade); + } else if (const auto* cross = std::get_if(&message)) { + handle_cross_trade(*cross); + } else if (const auto* directory = std::get_if(&message)) { + handle_stock_directory(*directory); + } +} + +template auto BookManager::handle_add_order(const AddOrderMessage&) -> void; +template auto BookManager::handle_add_order(const AddOrderMPIDAttributionMessage&) -> void; + +auto BookManager::book(std::uint16_t stock_locate) const -> const L3Book* { + const BookEntry* found = entry(stock_locate); + return found != nullptr ? &found->book : nullptr; +} + +auto BookManager::book_for_symbol(std::string_view symbol) const -> const L3Book* { + for (const auto& slot : m_books_by_locate) { + if (slot && slot->book.symbol() == symbol) { + return &slot->book; + } + } + return nullptr; +} + +auto BookManager::symbol_for_locate(std::uint16_t stock_locate) const -> std::string_view { + if (const BookEntry* found = entry(stock_locate); + found != nullptr && !found->book.symbol().empty()) { + return found->book.symbol(); + } + if (stock_locate < m_symbol_by_locate.size()) { + return m_symbol_by_locate[stock_locate]; + } + return {}; +} + +} // namespace itch::book diff --git a/src/book/l3_book.cpp b/src/book/l3_book.cpp new file mode 100644 index 0000000..a044028 --- /dev/null +++ b/src/book/l3_book.cpp @@ -0,0 +1,277 @@ +#include "itch/book/l3_book.hpp" + +#include +#include + +namespace itch::book { + +L3Book::L3Book(std::string symbol) : m_symbol {std::move(symbol)} {} + +auto L3Book::side_levels(Side side) noexcept -> std::vector& { + return side == Side::buy ? m_bids : m_asks; +} + +auto L3Book::side_levels(Side side) const noexcept -> const std::vector& { + return side == Side::buy ? m_bids : m_asks; +} + +auto L3Book::allocate_node() -> std::uint32_t { + if (m_free_head != NIL) { + const std::uint32_t index = m_free_head; + m_free_head = m_pool[index].next; + m_pool[index] = OrderNode {}; + return index; + } + m_pool.push_back(OrderNode {}); + return static_cast(m_pool.size() - 1); +} + +auto L3Book::free_node(std::uint32_t node_index) -> void { + m_pool[node_index] = OrderNode {}; + m_pool[node_index].next = m_free_head; + m_free_head = node_index; +} + +auto L3Book::find_level(Side side, std::uint32_t price) const -> std::uint32_t { + const auto& levels = side_levels(side); + // Bids are sorted descending, asks ascending; binary search accordingly. + // std::ranges::lower_bound would be the modern spelling, but it trips a + // known Clang/libstdc++ interop gap on some toolchains, so this sticks + // with the plain iterator-pair form for portability. + if (side == Side::buy) { + // NOLINTNEXTLINE(modernize-use-ranges) + const auto iter = std::lower_bound( + levels.begin(), + levels.end(), + price, + [](const Level& level, std::uint32_t value) { return level.price > value; } + ); + if (iter != levels.end() && iter->price == price) { + return static_cast(iter - levels.begin()); + } + } else { + // NOLINTNEXTLINE(modernize-use-ranges) + const auto iter = std::lower_bound( + levels.begin(), + levels.end(), + price, + [](const Level& level, std::uint32_t value) { return level.price < value; } + ); + if (iter != levels.end() && iter->price == price) { + return static_cast(iter - levels.begin()); + } + } + return NIL; +} + +auto L3Book::find_or_create_level(Side side, std::uint32_t price) -> std::uint32_t { + auto& levels = side_levels(side); + // See find_level's comment above on why these stay iterator-pair calls. + auto position = + side == Side::buy + ? std::lower_bound( // NOLINT(modernize-use-ranges) + levels.begin(), + levels.end(), + price, + [](const Level& level, std::uint32_t value) { return level.price > value; } + ) + : std::lower_bound( // NOLINT(modernize-use-ranges) + levels.begin(), + levels.end(), + price, + [](const Level& level, std::uint32_t value) { return level.price < value; } + ); + if (position != levels.end() && position->price == price) { + return static_cast(position - levels.begin()); + } + Level new_level {}; + new_level.price = price; + const auto inserted = levels.insert(position, new_level); + return static_cast(inserted - levels.begin()); +} + +auto L3Book::add_order( + std::uint64_t reference_number, Side side, std::uint32_t shares, std::uint32_t price +) -> void { + if (m_index.find(reference_number) != OrderIndex::NPOS) { + return; // Duplicate add; ignore to keep the book consistent. + } + const std::uint32_t node_index = allocate_node(); + OrderNode& node = m_pool[node_index]; + node.reference_number = reference_number; + node.shares = shares; + node.price = price; + node.side = side; + + const std::uint32_t level_index = find_or_create_level(side, price); + Level& level = side_levels(side)[level_index]; + + // Append to the tail of the level FIFO to preserve time priority. + node.prev = level.tail; + node.next = NIL; + if (level.tail != NIL) { + m_pool[level.tail].next = node_index; + } else { + level.head = node_index; + } + level.tail = node_index; + level.total_shares += shares; + ++level.order_count; + + m_index.insert(reference_number, node_index); +} + +auto L3Book::unlink_node(std::uint32_t node_index) -> void { + OrderNode& node = m_pool[node_index]; + const std::uint32_t level_index = find_level(node.side, node.price); + if (level_index == NIL) { + return; + } + auto& levels = side_levels(node.side); + Level& level = levels[level_index]; + + if (node.prev != NIL) { + m_pool[node.prev].next = node.next; + } else { + level.head = node.next; + } + if (node.next != NIL) { + m_pool[node.next].prev = node.prev; + } else { + level.tail = node.prev; + } + level.total_shares -= node.shares; + --level.order_count; + if (level.order_count == 0) { + levels.erase(levels.begin() + level_index); + } +} + +auto L3Book::execute_order(std::uint64_t reference_number, std::uint32_t shares) -> std::uint32_t { + const std::uint32_t node_index = m_index.find(reference_number); + if (node_index == OrderIndex::NPOS) { + return 0; + } + OrderNode& node = m_pool[node_index]; + const std::uint32_t removed = std::min(shares, node.shares); + + if (removed == node.shares) { + unlink_node(node_index); + free_node(node_index); + m_index.erase(reference_number); + return removed; + } + + node.shares -= removed; + const std::uint32_t level_index = find_level(node.side, node.price); + if (level_index != NIL) { + side_levels(node.side)[level_index].total_shares -= removed; + } + return removed; +} + +auto L3Book::reduce_order(std::uint64_t reference_number, std::uint32_t shares) -> std::uint32_t { + // A partial cancel removes shares exactly as an execution does. + return execute_order(reference_number, shares); +} + +auto L3Book::delete_order(std::uint64_t reference_number) -> void { + const std::uint32_t node_index = m_index.find(reference_number); + if (node_index == OrderIndex::NPOS) { + return; + } + unlink_node(node_index); + free_node(node_index); + m_index.erase(reference_number); +} + +auto L3Book::replace_order( + std::uint64_t old_reference_number, + std::uint64_t new_reference_number, + std::uint32_t shares, + std::uint32_t price +) -> void { + const std::uint32_t node_index = m_index.find(old_reference_number); + if (node_index == OrderIndex::NPOS) { + return; + } + const Side side = m_pool[node_index].side; + delete_order(old_reference_number); + add_order(new_reference_number, side, shares, price); +} + +auto L3Book::contains(std::uint64_t reference_number) const -> bool { + return m_index.contains(reference_number); +} + +auto L3Book::order_price(std::uint64_t reference_number) const -> std::optional { + const std::uint32_t node_index = m_index.find(reference_number); + if (node_index == OrderIndex::NPOS) { + return std::nullopt; + } + return m_pool[node_index].price; +} + +auto L3Book::order_side(std::uint64_t reference_number) const -> std::optional { + const std::uint32_t node_index = m_index.find(reference_number); + if (node_index == OrderIndex::NPOS) { + return std::nullopt; + } + return m_pool[node_index].side; +} + +auto L3Book::bbo() const -> Bbo { + Bbo result {}; + if (!m_bids.empty()) { + result.has_bid = true; + result.bid_price = StandardPrice {m_bids.front().price}; + result.bid_shares = m_bids.front().total_shares; + } + if (!m_asks.empty()) { + result.has_ask = true; + result.ask_price = StandardPrice {m_asks.front().price}; + result.ask_shares = m_asks.front().total_shares; + } + return result; +} + +auto L3Book::depth(Side side, std::size_t max_levels) const -> std::vector { + const auto& levels = side_levels(side); + const std::size_t count = max_levels == 0 ? levels.size() : std::min(max_levels, levels.size()); + std::vector result; + result.reserve(count); + for (std::size_t index = 0; index < count; ++index) { + result.push_back(DepthLevel { + .price = StandardPrice {levels[index].price}, + .shares = levels[index].total_shares, + .order_count = levels[index].order_count, + }); + } + return result; +} + +auto L3Book::orders_at(Side side, std::uint32_t price) const -> std::vector { + std::vector result; + const std::uint32_t level_index = find_level(side, price); + if (level_index == NIL) { + return result; + } + const Level& level = side_levels(side)[level_index]; + result.reserve(level.order_count); + for (std::uint32_t node_index = level.head; node_index != NIL; + node_index = m_pool[node_index].next) { + const OrderNode& node = m_pool[node_index]; + result.push_back(OrderView { + .reference_number = node.reference_number, + .shares = node.shares, + .price = StandardPrice {node.price}, + }); + } + return result; +} + +auto L3Book::level_count(Side side) const noexcept -> std::size_t { + return side_levels(side).size(); +} + +} // namespace itch::book diff --git a/src/encoder.cpp b/src/encoder.cpp new file mode 100644 index 0000000..97383d4 --- /dev/null +++ b/src/encoder.cpp @@ -0,0 +1,248 @@ +#include "itch/encoder.hpp" + +#include +#include +#include +#include + +#include "itch/parser.hpp" // utils::from_big_endian + +namespace itch { + +namespace { + +/// @brief Appends fields to a byte buffer in network (big-endian) order. +class ByteWriter { + public: + // Appends an integral field of width sizeof(IntType) in big-endian order. + template + auto put(IntType value) -> void { + const auto big = utils::from_big_endian(value); + std::array bytes {}; + std::memcpy(bytes.data(), &big, sizeof(IntType)); + m_bytes.insert(m_bytes.end(), bytes.begin(), bytes.end()); + } + + auto put_char(char value) -> void { m_bytes.push_back(static_cast(value)); } + + // Appends a fixed-width character field exactly as stored (no trimming). + auto put_chars(const char* source, std::size_t size) -> void { + for (std::size_t index = 0; index < size; ++index) { + m_bytes.push_back(static_cast(source[index])); + } + } + + // Appends the 48-bit ITCH timestamp (2-byte high, 4-byte low), big-endian. + auto put_timestamp(std::uint64_t timestamp) -> void { + constexpr int LOWER_SHIFT = 32; + put(static_cast(timestamp >> LOWER_SHIFT)); + put(static_cast(timestamp & 0xFFFFFFFFULL)); + } + + [[nodiscard]] auto take() -> std::vector { return std::move(m_bytes); } + + private: + std::vector m_bytes; +}; + +// Writes the header common to every message: type, locate, tracking, timestamp. +template +auto write_header(ByteWriter& writer, const MsgType& msg) -> void { + writer.put_char(msg.message_type); + writer.put(msg.stock_locate); + writer.put(msg.tracking_number); + writer.put_timestamp(msg.timestamp); +} + +auto write_body(ByteWriter& writer, const SystemEventMessage& msg) -> void { + writer.put_char(msg.event_code); +} + +auto write_body(ByteWriter& writer, const StockDirectoryMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.market_category); + writer.put_char(msg.financial_status_indicator); + writer.put(msg.round_lot_size); + writer.put_char(msg.round_lots_only); + writer.put_char(msg.issue_classification); + writer.put_chars(msg.issue_sub_type, 2); + writer.put_char(msg.authenticity); + writer.put_char(msg.short_sale_threshold_indicator); + writer.put_char(msg.ipo_flag); + writer.put_char(msg.luld_ref); + writer.put_char(msg.etp_flag); + writer.put(msg.etp_leverage_factor); + writer.put_char(msg.inverse_indicator); +} + +auto write_body(ByteWriter& writer, const StockTradingActionMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.trading_state); + writer.put_char(msg.reserved); + writer.put_chars(msg.reason, 4); +} + +auto write_body(ByteWriter& writer, const RegSHOMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.reg_sho_action); +} + +auto write_body(ByteWriter& writer, const MarketParticipantPositionMessage& msg) -> void { + writer.put_chars(msg.mpid, 4); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.primary_market_maker); + writer.put_char(msg.market_maker_mode); + writer.put_char(msg.market_participant_state); +} + +auto write_body(ByteWriter& writer, const MWCBDeclineLevelMessage& msg) -> void { + writer.put(msg.level1); + writer.put(msg.level2); + writer.put(msg.level3); +} + +auto write_body(ByteWriter& writer, const MWCBStatusMessage& msg) -> void { + writer.put_char(msg.breached_level); +} + +auto write_body(ByteWriter& writer, const IPOQuotingPeriodUpdateMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.ipo_quotation_release_time); + writer.put_char(msg.ipo_quotation_release_qualifier); + writer.put(msg.ipo_price); +} + +auto write_body(ByteWriter& writer, const LULDAuctionCollarMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.auction_collar_reference_price); + writer.put(msg.upper_auction_collar_price); + writer.put(msg.lower_auction_collar_price); + writer.put(msg.auction_collar_extension); +} + +auto write_body(ByteWriter& writer, const OperationalHaltMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.market_code); + writer.put_char(msg.operational_halt_action); +} + +auto write_body(ByteWriter& writer, const AddOrderMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put_char(msg.buy_sell_indicator); + writer.put(msg.shares); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.price); +} + +auto write_body(ByteWriter& writer, const AddOrderMPIDAttributionMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put_char(msg.buy_sell_indicator); + writer.put(msg.shares); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.price); + writer.put_chars(msg.attribution, 4); +} + +auto write_body(ByteWriter& writer, const OrderExecutedMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put(msg.executed_shares); + writer.put(msg.match_number); +} + +auto write_body(ByteWriter& writer, const OrderExecutedWithPriceMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put(msg.executed_shares); + writer.put(msg.match_number); + writer.put_char(msg.printable); + writer.put(msg.execution_price); +} + +auto write_body(ByteWriter& writer, const OrderCancelMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put(msg.cancelled_shares); +} + +auto write_body(ByteWriter& writer, const OrderDeleteMessage& msg) -> void { + writer.put(msg.order_reference_number); +} + +auto write_body(ByteWriter& writer, const OrderReplaceMessage& msg) -> void { + writer.put(msg.original_order_reference_number); + writer.put(msg.new_order_reference_number); + writer.put(msg.shares); + writer.put(msg.price); +} + +auto write_body(ByteWriter& writer, const NonCrossTradeMessage& msg) -> void { + writer.put(msg.order_reference_number); + writer.put_char(msg.buy_sell_indicator); + writer.put(msg.shares); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.price); + writer.put(msg.match_number); +} + +auto write_body(ByteWriter& writer, const CrossTradeMessage& msg) -> void { + writer.put(msg.shares); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.cross_price); + writer.put(msg.match_number); + writer.put_char(msg.cross_type); +} + +auto write_body(ByteWriter& writer, const BrokenTradeMessage& msg) -> void { + writer.put(msg.match_number); +} + +auto write_body(ByteWriter& writer, const NOIIMessage& msg) -> void { + writer.put(msg.paired_shares); + writer.put(msg.imbalance_shares); + writer.put_char(msg.imbalance_direction); + writer.put_chars(msg.stock, STOCK_LEN); + writer.put(msg.far_price); + writer.put(msg.near_price); + writer.put(msg.current_reference_price); + writer.put_char(msg.cross_type); + writer.put_char(msg.price_variation_indicator); +} + +auto write_body(ByteWriter& writer, const RetailPriceImprovementIndicatorMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.interest_flag); +} + +auto write_body(ByteWriter& writer, const DLCRMessage& msg) -> void { + writer.put_chars(msg.stock, STOCK_LEN); + writer.put_char(msg.open_eligibility_status); + writer.put(msg.minimum_allowable_price); + writer.put(msg.maximum_allowable_price); + writer.put(msg.near_execution_price); + writer.put(msg.near_execution_time); + writer.put(msg.lower_price_range_collar); + writer.put(msg.upper_price_range_collar); +} + +} // namespace + +auto encode_message(const Message& message) -> std::vector { + ByteWriter writer; + std::visit( + [&writer](const auto& concrete) { + write_header(writer, concrete); + write_body(writer, concrete); + }, + message + ); + return writer.take(); +} + +auto encode_frame(const Message& message) -> std::vector { + const std::vector body = encode_message(message); + ByteWriter writer; + writer.put(static_cast(body.size())); + std::vector frame = writer.take(); + frame.insert(frame.end(), body.begin(), body.end()); + return frame; +} + +} // namespace itch diff --git a/src/io/arrow_export.cpp b/src/io/arrow_export.cpp new file mode 100644 index 0000000..1cbb033 --- /dev/null +++ b/src/io/arrow_export.cpp @@ -0,0 +1,257 @@ +#include "itch/io/arrow_export.hpp" + +#ifdef ITCH_WITH_ARROW + +#include +#include +#include + +#include +#include + +#include "itch/price.hpp" + +namespace itch::io { + +struct ArrowExporter::Impl { + arrow::StringBuilder message_type; + arrow::UInt64Builder timestamp; + arrow::UInt16Builder stock_locate; + arrow::UInt16Builder tracking_number; + arrow::StringBuilder symbol; + arrow::UInt64Builder reference_number; + arrow::StringBuilder side; + arrow::UInt64Builder shares; + arrow::DoubleBuilder price; + arrow::UInt64Builder match_number; + arrow::StringBuilder printable; + arrow::StringBuilder extra; + std::uint64_t row_count {0}; + std::string last_error; + + // Builds the common header columns shared by every message type. + template + auto append_common(const MsgType& msg) -> void { + (void)message_type.Append(std::string {msg.message_type}); + (void)timestamp.Append(msg.timestamp); + (void)stock_locate.Append(msg.stock_locate); + (void)tracking_number.Append(msg.tracking_number); + } + + // Appends nulls/empties for the type-specific columns, overwritten as needed. + auto append_blank_specific() -> void { + (void)symbol.AppendEmptyValue(); + (void)reference_number.AppendNull(); + (void)side.AppendEmptyValue(); + (void)shares.AppendNull(); + (void)price.AppendNull(); + (void)match_number.AppendNull(); + (void)printable.AppendEmptyValue(); + (void)extra.AppendEmptyValue(); + } +}; + +ArrowExporter::ArrowExporter() : m_impl {std::make_unique()} {} +ArrowExporter::~ArrowExporter() = default; + +auto ArrowExporter::rows() const noexcept -> std::uint64_t { return m_impl->row_count; } +auto ArrowExporter::error() const -> const std::string& { return m_impl->last_error; } + +auto ArrowExporter::append(const Message& message) -> void { + Impl& impl = *m_impl; + std::visit([&impl](const auto& msg) { impl.append_common(msg); }, message); + + // Default every type-specific column to null/empty, then set what applies. + // Replacing the convenience of AppendEmptyValue with explicit per-type code + // keeps the column lengths in lockstep with the header columns. + std::string symbol_value; + bool has_reference = false; + std::uint64_t reference = 0; + std::string side_value; + bool has_shares = false; + std::uint64_t shares_value = 0; + bool has_price = false; + double price_value = 0.0; + bool has_match = false; + std::uint64_t match_value = 0; + std::string printable_value; + std::string extra_value; + + if (const auto* add = std::get_if(&message)) { + symbol_value = to_string(add->stock, STOCK_LEN); + has_reference = true; + reference = add->order_reference_number; + side_value = std::string {add->buy_sell_indicator}; + has_shares = true; + shares_value = add->shares; + has_price = true; + price_value = StandardPrice {add->price}.to_double(); + } else if (const auto* add_mpid = std::get_if(&message)) { + symbol_value = to_string(add_mpid->stock, STOCK_LEN); + has_reference = true; + reference = add_mpid->order_reference_number; + side_value = std::string {add_mpid->buy_sell_indicator}; + has_shares = true; + shares_value = add_mpid->shares; + has_price = true; + price_value = StandardPrice {add_mpid->price}.to_double(); + extra_value = "mpid=" + to_string(add_mpid->attribution, 4); + } else if (const auto* exec = std::get_if(&message)) { + has_reference = true; + reference = exec->order_reference_number; + has_shares = true; + shares_value = exec->executed_shares; + has_match = true; + match_value = exec->match_number; + } else if (const auto* exec_px = std::get_if(&message)) { + has_reference = true; + reference = exec_px->order_reference_number; + has_shares = true; + shares_value = exec_px->executed_shares; + has_price = true; + price_value = StandardPrice {exec_px->execution_price}.to_double(); + has_match = true; + match_value = exec_px->match_number; + printable_value = std::string {exec_px->printable}; + } else if (const auto* cancel = std::get_if(&message)) { + has_reference = true; + reference = cancel->order_reference_number; + has_shares = true; + shares_value = cancel->cancelled_shares; + } else if (const auto* del = std::get_if(&message)) { + has_reference = true; + reference = del->order_reference_number; + } else if (const auto* replace = std::get_if(&message)) { + has_reference = true; + reference = replace->new_order_reference_number; + has_shares = true; + shares_value = replace->shares; + has_price = true; + price_value = StandardPrice {replace->price}.to_double(); + extra_value = "orig=" + std::to_string(replace->original_order_reference_number); + } else if (const auto* trade = std::get_if(&message)) { + symbol_value = to_string(trade->stock, STOCK_LEN); + has_reference = true; + reference = trade->order_reference_number; + side_value = std::string {trade->buy_sell_indicator}; + has_shares = true; + shares_value = trade->shares; + has_price = true; + price_value = StandardPrice {trade->price}.to_double(); + has_match = true; + match_value = trade->match_number; + } else if (const auto* cross = std::get_if(&message)) { + symbol_value = to_string(cross->stock, STOCK_LEN); + has_shares = true; + shares_value = cross->shares; + has_price = true; + price_value = StandardPrice {cross->cross_price}.to_double(); + has_match = true; + match_value = cross->match_number; + extra_value = std::string {"cross="} + cross->cross_type; + } else if (const auto* event = std::get_if(&message)) { + extra_value = std::string {"event="} + event->event_code; + } else if (const auto* directory = std::get_if(&message)) { + symbol_value = to_string(directory->stock, STOCK_LEN); + } + + (void)impl.symbol.Append(symbol_value); + if (has_reference) { + (void)impl.reference_number.Append(reference); + } else { + (void)impl.reference_number.AppendNull(); + } + (void)impl.side.Append(side_value); + if (has_shares) { + (void)impl.shares.Append(shares_value); + } else { + (void)impl.shares.AppendNull(); + } + if (has_price) { + (void)impl.price.Append(price_value); + } else { + (void)impl.price.AppendNull(); + } + if (has_match) { + (void)impl.match_number.Append(match_value); + } else { + (void)impl.match_number.AppendNull(); + } + (void)impl.printable.Append(printable_value); + (void)impl.extra.Append(extra_value); + ++impl.row_count; +} + +auto ArrowExporter::write_parquet(const std::string& path) -> bool { + Impl& impl = *m_impl; + + auto finish = + [&impl](arrow::ArrayBuilder& builder, std::shared_ptr& out) -> bool { + const arrow::Status status = builder.Finish(&out); + if (!status.ok()) { + impl.last_error = status.ToString(); + return false; + } + return true; + }; + + std::shared_ptr columns[12]; + arrow::ArrayBuilder* builders[12] = { + &impl.message_type, + &impl.timestamp, + &impl.stock_locate, + &impl.tracking_number, + &impl.symbol, + &impl.reference_number, + &impl.side, + &impl.shares, + &impl.price, + &impl.match_number, + &impl.printable, + &impl.extra + }; + for (int index = 0; index < 12; ++index) { + if (!finish(*builders[index], columns[index])) { + return false; + } + } + + auto schema = arrow::schema({ + arrow::field("message_type", arrow::utf8()), + arrow::field("timestamp", arrow::uint64()), + arrow::field("stock_locate", arrow::uint16()), + arrow::field("tracking_number", arrow::uint16()), + arrow::field("symbol", arrow::utf8()), + arrow::field("reference_number", arrow::uint64()), + arrow::field("side", arrow::utf8()), + arrow::field("shares", arrow::uint64()), + arrow::field("price", arrow::float64()), + arrow::field("match_number", arrow::uint64()), + arrow::field("printable", arrow::utf8()), + arrow::field("extra", arrow::utf8()), + }); + + std::vector> column_vector {columns, columns + 12}; + const auto table = arrow::Table::Make(schema, column_vector); + + auto outfile_result = arrow::io::FileOutputStream::Open(path); + if (!outfile_result.ok()) { + impl.last_error = outfile_result.status().ToString(); + return false; + } + const auto status = parquet::arrow::WriteTable( + *table, + arrow::default_memory_pool(), + *outfile_result, + impl.row_count > 0 ? impl.row_count : 1 + ); + if (!status.ok()) { + impl.last_error = status.ToString(); + return false; + } + return true; +} + +} // namespace itch::io + +#endif // ITCH_WITH_ARROW diff --git a/src/io/csv_sink.cpp b/src/io/csv_sink.cpp new file mode 100644 index 0000000..69e00ca --- /dev/null +++ b/src/io/csv_sink.cpp @@ -0,0 +1,144 @@ +#include "itch/io/csv_sink.hpp" + +#include +#include +#include + +#include "itch/price.hpp" + +namespace itch::io { + +namespace { + +/// @brief The flattened CSV view of one message, with blanks for absent fields. +struct Row { + char message_type {'\0'}; + std::uint64_t timestamp {0}; + std::uint16_t stock_locate {0}; + std::uint16_t tracking_number {0}; + std::string symbol; + std::string reference_number; + char side {'\0'}; + std::string shares; + std::string price; + std::string match_number; + char printable {'\0'}; + std::string extra; +}; + +// Builds the common header fields shared by every message. +template +auto fill_common(Row& row, const MsgType& msg) -> void { + row.message_type = msg.message_type; + row.timestamp = msg.timestamp; + row.stock_locate = msg.stock_locate; + row.tracking_number = msg.tracking_number; +} + +// Quotes a value only if it contains a comma; symbols never do, but be safe. +auto field(const std::string& value) -> std::string { + // std::string::contains needs C++23, but this file must also build under + // this project's C++20 floor. + // NOLINTNEXTLINE(readability-container-contains) + if (value.find(',') != std::string::npos) { + return std::format("\"{}\"", value); + } + return value; +} + +auto build_row(const Message& message) -> Row { + Row row {}; + std::visit([&row](const auto& msg) { fill_common(row, msg); }, message); + + if (const auto* add = std::get_if(&message)) { + row.symbol = to_string(add->stock, STOCK_LEN); + row.reference_number = std::to_string(add->order_reference_number); + row.side = add->buy_sell_indicator; + row.shares = std::to_string(add->shares); + row.price = StandardPrice {add->price}.to_string(); + } else if (const auto* add_mpid = std::get_if(&message)) { + row.symbol = to_string(add_mpid->stock, STOCK_LEN); + row.reference_number = std::to_string(add_mpid->order_reference_number); + row.side = add_mpid->buy_sell_indicator; + row.shares = std::to_string(add_mpid->shares); + row.price = StandardPrice {add_mpid->price}.to_string(); + row.extra = std::format("mpid={}", to_string(add_mpid->attribution, 4)); + } else if (const auto* exec = std::get_if(&message)) { + row.reference_number = std::to_string(exec->order_reference_number); + row.shares = std::to_string(exec->executed_shares); + row.match_number = std::to_string(exec->match_number); + } else if (const auto* exec_px = std::get_if(&message)) { + row.reference_number = std::to_string(exec_px->order_reference_number); + row.shares = std::to_string(exec_px->executed_shares); + row.price = StandardPrice {exec_px->execution_price}.to_string(); + row.match_number = std::to_string(exec_px->match_number); + row.printable = exec_px->printable; + } else if (const auto* cancel = std::get_if(&message)) { + row.reference_number = std::to_string(cancel->order_reference_number); + row.shares = std::to_string(cancel->cancelled_shares); + } else if (const auto* del = std::get_if(&message)) { + row.reference_number = std::to_string(del->order_reference_number); + } else if (const auto* replace = std::get_if(&message)) { + row.reference_number = std::to_string(replace->new_order_reference_number); + row.shares = std::to_string(replace->shares); + row.price = StandardPrice {replace->price}.to_string(); + row.extra = std::format("orig={}", replace->original_order_reference_number); + } else if (const auto* trade = std::get_if(&message)) { + row.symbol = to_string(trade->stock, STOCK_LEN); + row.reference_number = std::to_string(trade->order_reference_number); + row.side = trade->buy_sell_indicator; + row.shares = std::to_string(trade->shares); + row.price = StandardPrice {trade->price}.to_string(); + row.match_number = std::to_string(trade->match_number); + } else if (const auto* cross = std::get_if(&message)) { + row.symbol = to_string(cross->stock, STOCK_LEN); + row.shares = std::to_string(cross->shares); + row.price = StandardPrice {cross->cross_price}.to_string(); + row.match_number = std::to_string(cross->match_number); + row.extra = std::format("cross={}", cross->cross_type); + } else if (const auto* event = std::get_if(&message)) { + row.extra = std::format("event={}", event->event_code); + } else if (const auto* directory = std::get_if(&message)) { + row.symbol = to_string(directory->stock, STOCK_LEN); + row.extra = std::format("category={}", directory->market_category); + } + return row; +} + +// Renders a single character field, blank when unset. +auto char_field(char value) -> std::string { + return value == '\0' ? std::string {} : std::string {value}; +} + +} // namespace + +CsvSink::CsvSink(std::ostream& out, bool write_header) : m_out {out} { + if (write_header) { + m_out << "message_type,timestamp,stock_locate,tracking_number,symbol,reference_number," + "side,shares,price,match_number,printable,extra\n"; + } +} + +auto CsvSink::write(const Message& message) -> void { + const Row row = build_row(message); + m_out << std::format( + "{},{},{},{},{},{},{},{},{},{},{},{}\n", + row.message_type, + row.timestamp, + row.stock_locate, + row.tracking_number, + field(row.symbol), + row.reference_number, + char_field(row.side), + row.shares, + row.price, + row.match_number, + char_field(row.printable), + field(row.extra) + ); + ++m_rows_written; +} + +auto CsvSink::flush() -> void { m_out.flush(); } + +} // namespace itch::io diff --git a/src/order_book.cpp b/src/order_book.cpp index f41a925..bad4cea 100644 --- a/src/order_book.cpp +++ b/src/order_book.cpp @@ -1,18 +1,23 @@ #include "itch/order_book.hpp" #include +#include +#include +#include #include #include +#include "itch/price.hpp" + namespace itch { -auto PriceLevel::add_order(std::shared_ptr order) -> void { +auto PriceLevel::add_order(const std::shared_ptr& order) -> void { total_shares += order->shares; orders.push_back(order); order->level = this; } -auto PriceLevel::remove_order(OrderIt order_it) -> void { orders.erase(order_it); } +auto PriceLevel::remove_order(const OrderIt& order_it) -> void { orders.erase(order_it); } auto LimitOrderBook::process(const Message& message) -> void { std::visit( @@ -27,44 +32,40 @@ auto LimitOrderBook::process(const Message& message) -> void { } auto LimitOrderBook::print(std::ostream& out, unsigned int delay_ms) const -> void { - std::ios state(nullptr); - state.copyfmt(out); - out << std::fixed << std::setprecision(4); + constexpr std::string_view BORDER = "==========================================\n"; + constexpr std::string_view MID_RULE = "-----------+--------------+--------------\n"; - out << "==========================================\n"; + out << BORDER; out << " SHARES | PRICE | SIDE \n"; - out << "==========================================\n"; + out << BORDER; - if (!m_asks.empty()) { - for (auto iter = m_asks.rbegin(); iter != m_asks.rend(); ++iter) { + const auto print_level = + [&](std::uint32_t total_shares, std::uint32_t raw_price, std::string_view side) { if (delay_ms > 0) { std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); } - double price = iter->first / PRICE_DIVISOR; - out << std::setw(10) << iter->second.total_shares << " | " << std::setw(12) << price - << " | Ask\n"; - if (delay_ms > 0) + out << std::format( + "{:>10} | {:>12.4f} | {}\n", total_shares, make_price(raw_price).to_double(), side + ); + if (delay_ms > 0) { out << std::flush; - } + } + }; + + // Asks are stored low-to-high; print them high-to-low so the best ask sits + // just above the spread. + for (const auto& [raw_price, level] : std::ranges::reverse_view(m_asks)) { + print_level(level.total_shares, raw_price, "Ask"); } - out << "-----------+--------------+--------------\n"; + out << MID_RULE; - if (!m_bids.empty()) { - for (const auto& [raw_price, level] : m_bids) { - if (delay_ms > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms)); - } - double price = raw_price / PRICE_DIVISOR; - out << std::setw(10) << level.total_shares << " | " << std::setw(12) << price - << " | Bid\n"; - if (delay_ms > 0) - out << std::flush; - } + // Bids are stored high-to-low, which is already the desired print order. + for (const auto& [raw_price, level] : m_bids) { + print_level(level.total_shares, raw_price, "Bid"); } - out << "==========================================\n"; - out.copyfmt(state); + out << BORDER; } auto LimitOrderBook::handle_message(const AddOrderMessage& msg) -> void { @@ -94,24 +95,24 @@ auto LimitOrderBook::handle_message(const AddOrderMPIDAttributionMessage& msg) - } auto LimitOrderBook::handle_message(const OrderExecutedMessage& msg) -> void { - auto it = m_orders.find(msg.order_reference_number); - if (it == m_orders.end() || (*it->second)->stock != m_stock_symbol) { + auto order_iter = m_orders.find(msg.order_reference_number); + if (order_iter == m_orders.end() || (*order_iter->second)->stock != m_stock_symbol) { return; } remove_order(msg.order_reference_number, msg.executed_shares); } auto LimitOrderBook::handle_message(const OrderExecutedWithPriceMessage& msg) -> void { - auto it = m_orders.find(msg.order_reference_number); - if (it == m_orders.end() || (*it->second)->stock != m_stock_symbol) { + auto order_iter = m_orders.find(msg.order_reference_number); + if (order_iter == m_orders.end() || (*order_iter->second)->stock != m_stock_symbol) { return; } remove_order(msg.order_reference_number, msg.executed_shares); } auto LimitOrderBook::handle_message(const OrderCancelMessage& msg) -> void { - auto it = m_orders.find(msg.order_reference_number); - if (it == m_orders.end() || (*it->second)->stock != m_stock_symbol) { + auto order_iter = m_orders.find(msg.order_reference_number); + if (order_iter == m_orders.end() || (*order_iter->second)->stock != m_stock_symbol) { return; } remove_order(msg.order_reference_number, msg.cancelled_shares); @@ -159,9 +160,11 @@ auto LimitOrderBook::remove_order(uint64_t order_ref, uint32_t canceled_shares) return; } - OrderIt order_iterator = iter->second; - auto order_ptr = *order_iterator; - PriceLevel* level = order_ptr->level; + auto order_iterator = iter->second; + // This copy is deliberate: it keeps the Order alive after the list node is + // erased below, since the order's price and side are read afterwards. + auto order_ptr = *order_iterator; // NOLINT(performance-unnecessary-copy-initialization) + PriceLevel* level = order_ptr->level; level->total_shares -= canceled_shares; order_ptr->shares -= canceled_shares; diff --git a/src/parser.cpp b/src/parser.cpp index b74e10e..8aee50d 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -1,68 +1,50 @@ #include "itch/parser.hpp" +#include +#include #include #include #include #include +#include #include +#include "itch/detail/wire.hpp" + namespace itch { namespace utils { -inline auto is_little_endian() -> bool { -#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ - return true; -#elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ - return false; -#elif defined(_WIN32) // Windows is always little-endian - return true; -#else - // Runtime check if compile-time check is not available - const union { - uint32_t i; - char c[4]; - } bint = {0x01020304}; - return bint.c[0] == 4; -#endif -} -template -auto from_big_endian(T value) -> T { - if (is_little_endian()) { - return swap_bytes(value); - } - return value; -} - -template -auto unpack(const char* buffer, size_t& offset) -> T { - T value; - std::memcpy(&value, buffer + offset, sizeof(T)); - offset += sizeof(T); +template +auto unpack(const char* buffer, std::size_t& offset) -> ValueType { + ValueType value; + std::memcpy(&value, buffer + offset, sizeof(ValueType)); + offset += sizeof(ValueType); - if constexpr (std::is_integral_v && sizeof(T) > 1) { + if constexpr (std::is_integral_v && sizeof(ValueType) > 1) { return from_big_endian(value); } else { return value; } } -inline auto unpack_string(const char* buffer, size_t& offset, char* dest, size_t size) -> void { +inline auto unpack_string(const char* buffer, std::size_t& offset, char* dest, std::size_t size) + -> void { std::memcpy(dest, buffer + offset, size); offset += size; } -inline auto unpack_timestamp(const char* buffer, size_t& offset) -> uint64_t { - uint16_t high {}; - uint32_t low {}; - constexpr int lower_shift = 32; +inline auto unpack_timestamp(const char* buffer, std::size_t& offset) -> std::uint64_t { + std::uint16_t high {}; + std::uint32_t low {}; + constexpr int LOWER_SHIFT = 32; std::memcpy(&high, buffer + offset, sizeof(high)); offset += sizeof(high); std::memcpy(&low, buffer + offset, sizeof(low)); offset += sizeof(low); high = from_big_endian(high); low = from_big_endian(low); - return (static_cast(high) << lower_shift) | low; + return (static_cast(high) << LOWER_SHIFT) | low; } } // namespace utils @@ -246,83 +228,127 @@ auto unpack_message(DLCRMessage& msg, const char* buffer, size_t& offset) -> voi msg.upper_price_range_collar = utils::unpack(buffer, offset); } -template -auto Parser::register_handler(char type) -> void { - m_handlers[type] = [](const char* buffer) -> Message { - T msg; - size_t offset = 1; // Skip message type +namespace { + +using detail::WIRE_SIZE; + +/// @brief Decodes the common header and type-specific body of a message of +/// type MsgType from a frame, returning it wrapped in the Message variant. +template +auto decode_typed(const char* buffer) -> Message { + MsgType msg; + std::size_t offset = 1; // Skip the message type byte. + + msg.stock_locate = utils::unpack(buffer, offset); + msg.tracking_number = utils::unpack(buffer, offset); + msg.timestamp = utils::unpack_timestamp(buffer, offset); + + unpack_message(msg, buffer, offset); + + return msg; +} + +/// @brief One slot of the flat, type-byte-indexed dispatch table. +struct DispatchEntry { + std::uint16_t wire_size {0}; ///< Expected on-wire frame size (0 == unknown type). + Message (*decode)(const char*) {nullptr}; ///< Decoder, or null for an unknown type. +}; - msg.stock_locate = utils::unpack(buffer, offset); - msg.tracking_number = utils::unpack(buffer, offset); - msg.timestamp = utils::unpack_timestamp(buffer, offset); +constexpr std::size_t DISPATCH_TABLE_SIZE = 256; - unpack_message(msg, buffer, offset); +/// @brief Builds the dispatch table at compile time so message dispatch is a +/// single flat-array lookup plus a direct (inlinable) call, with no map +/// traversal and no type-erased `std::function` indirection. +consteval auto build_dispatch_table() -> std::array { + std::array table {}; - return msg; + auto add = [&table](char type) { + table[static_cast(type)] = + DispatchEntry {static_cast(WIRE_SIZE), &decode_typed}; }; + + // The canonical message-type registry lives in itch/detail/wire.hpp so the + // dispatch table, overlay, and encoder all share one definition. + detail::for_each_message_type(add); + + return table; } -Parser::Parser() { - register_handler('S'); - register_handler('R'); - register_handler('H'); - register_handler('Y'); - register_handler('L'); - register_handler('V'); - register_handler('W'); - register_handler('K'); - register_handler('J'); - register_handler('h'); - register_handler('A'); - register_handler('F'); - register_handler('E'); - register_handler('C'); - register_handler('X'); - register_handler('D'); - register_handler('U'); - register_handler('P'); - register_handler('Q'); - register_handler('B'); - register_handler('I'); - register_handler('N'); - register_handler('O'); +constexpr auto DISPATCH_TABLE = build_dispatch_table(); + +} // namespace + +auto Parser::report_error(ParseError error, char message_type) -> void { + switch (error) { + case ParseError::unknown_type: + ++m_unknown_message_count; + break; + case ParseError::size_mismatch: + case ParseError::truncated: + ++m_malformed_message_count; + break; + } + if (m_error_callback) { + m_error_callback(error, message_type); + } } -auto Parser::parse(const char* data, size_t size, const MessageCallback& callback) -> void { - size_t offset = 0; +auto Parser::parse_impl(const char* data, std::size_t size, const MessageCallback& callback) + -> std::optional { + std::size_t offset = 0; while (offset < size) { - // Ensure we can read the length field - if (offset + sizeof(uint16_t) > size) { - throw std::runtime_error("Incomplete message header at end of buffer."); + // Ensure we can read the 2-byte length prefix. + if (offset + sizeof(std::uint16_t) > size) { + report_error(ParseError::truncated, '\0'); + return ParseError::truncated; } - uint16_t length {}; + std::uint16_t length {}; std::memcpy(&length, data + offset, sizeof(length)); length = utils::from_big_endian(length); - offset += sizeof(uint16_t); + offset += sizeof(std::uint16_t); if (length == 0) { - continue; + continue; // Skip zero-length padding frames. } - // Ensure the full message payload is available + // Ensure the full declared payload is present. if (offset + length > size) { - throw std::runtime_error("Incomplete message at end of buffer."); + report_error(ParseError::truncated, '\0'); + return ParseError::truncated; } + const char* message = data + offset; const char message_type = message[0]; + offset += length; - auto handler_it = m_handlers.find(message_type); - if (handler_it != m_handlers.end()) { - callback(handler_it->second(message)); - } else { - std::cerr << "Unknown or unhandled message type: " << message_type << '\n'; + const DispatchEntry& entry = DISPATCH_TABLE[static_cast(message_type)]; + if (entry.decode == nullptr) { + // Unknown type: skip and count rather than writing to a global stream. + report_error(ParseError::unknown_type, message_type); + continue; } - offset += length; + // A frame shorter than the type requires would make the decoder read into + // the next frame; reject it. A longer frame is tolerated for forward + // compatibility (the trailing bytes are simply skipped). + if (length < entry.wire_size) { + report_error(ParseError::size_mismatch, message_type); + continue; + } + + callback(entry.decode(message)); + } + return std::nullopt; +} + +auto Parser::parse(const char* data, size_t size, const MessageCallback& callback) -> void { + if (parse_impl(data, size, callback).has_value()) { + throw std::runtime_error("Incomplete message at end of buffer."); } } constexpr size_t average_message_size = 20; -auto Parser::parse(const char* data, size_t size) -> std::vector { + +auto Parser::parse(const char* data, size_t size) -> std::vector { std::vector messages; messages.reserve(size / average_message_size); parse(data, size, [&](const Message& msg) { messages.push_back(msg); }); @@ -349,6 +375,56 @@ auto Parser::parse(const char* data, size_t size, const std::vector& messa return results; } +namespace { +// std::byte and char may legitimately alias; route through void* to obtain the +// char view the framing core works with, without a forbidden reinterpret_cast. +auto as_char_ptr(std::span data) -> const char* { + // char may alias std::byte; the house style forbids reinterpret_cast, so the + // void* hop is the intended form here. + const void* raw = data.data(); + return static_cast(raw); // NOLINT(bugprone-casting-through-void) +} +} // namespace + +auto Parser::parse(std::span data, const MessageCallback& callback) -> void { + parse(as_char_ptr(data), data.size(), callback); +} + +auto Parser::parse(std::span data) -> std::vector { + return parse(as_char_ptr(data), data.size()); +} + +auto Parser::parse(std::span data, const std::vector& messages) + -> std::vector { + return parse(as_char_ptr(data), data.size(), messages); +} + +#ifdef __cpp_lib_expected +auto Parser::try_parse(std::span data, const MessageCallback& callback) + -> std::expected { + if (auto error = parse_impl(as_char_ptr(data), data.size(), callback)) { + return std::unexpected(*error); + } + return {}; +} + +auto Parser::try_parse(std::span data +) -> std::expected, ParseError> { + std::vector messages; + messages.reserve(data.size() / average_message_size); + if (auto error = parse_impl(as_char_ptr(data), data.size(), [&](const Message& msg) { + messages.push_back(msg); + })) { + return std::unexpected(*error); + } + return messages; +} +#endif + +auto Parser::set_error_callback(ErrorCallback callback) -> void { + m_error_callback = std::move(callback); +} + // Read the whole stream into a buffer static auto read_stream_into_buffer(std::istream& data) -> std::vector { data.seekg(0, std::ios::end); diff --git a/src/replay.cpp b/src/replay.cpp new file mode 100644 index 0000000..d67f97d --- /dev/null +++ b/src/replay.cpp @@ -0,0 +1,52 @@ +#include "itch/replay.hpp" + +#include +#include +#include + +namespace itch { + +namespace { + +// Extracts the nanoseconds-past-midnight timestamp from any message. +auto timestamp_of(const Message& message) -> std::uint64_t { + return std::visit([](const auto& concrete) { return concrete.timestamp; }, message); +} + +} // namespace + +auto ReplayEngine::replay(std::span data, const MessageCallback& callback) const + -> std::uint64_t { + using clock = std::chrono::steady_clock; + const bool paced = m_speed_multiplier > 0.0; + + std::uint64_t replayed = 0; + bool have_base = false; + std::uint64_t base_timestamp = 0; + clock::time_point base_wall {}; + + Parser parser; + parser.parse(data, [&](const Message& message) { + if (paced) { + const std::uint64_t feed_timestamp = timestamp_of(message); + if (!have_base) { + base_timestamp = feed_timestamp; + base_wall = clock::now(); + have_base = true; + } else if (feed_timestamp > base_timestamp) { + // Scale the elapsed feed time by the speed multiplier and sleep + // until that point relative to the first message's wall time. + const double elapsed_ns = + static_cast(feed_timestamp - base_timestamp) / m_speed_multiplier; + const auto target = + base_wall + std::chrono::nanoseconds {static_cast(elapsed_ns)}; + std::this_thread::sleep_until(target); + } + } + callback(message); + ++replayed; + }); + return replayed; +} + +} // namespace itch diff --git a/src/transport/moldudp64.cpp b/src/transport/moldudp64.cpp new file mode 100644 index 0000000..a01e02c --- /dev/null +++ b/src/transport/moldudp64.cpp @@ -0,0 +1,75 @@ +#include "itch/transport/moldudp64.hpp" + +#include +#include +#include + +namespace itch::transport { + +namespace { + +// Reads a big-endian unsigned integer of the field's width out of a byte span. +template +auto read_big_endian(std::span data, std::size_t offset) -> UintType { + UintType value {}; + std::memcpy(&value, data.data() + offset, sizeof(UintType)); + return utils::from_big_endian(value); +} + +} // namespace + +MoldUdp64Decoder::MoldUdp64Decoder(MessageCallback callback) : m_callback {std::move(callback)} {} + +auto MoldUdp64Decoder::decode_packet(std::span packet +) -> std::optional { + if (packet.size() < HEADER_SIZE) { + return std::nullopt; + } + + ++m_packets_decoded; + + MoldUdp64Header header {}; + std::memcpy(header.session.data(), packet.data(), header.session.size()); + constexpr std::size_t SEQUENCE_OFFSET = 10; + constexpr std::size_t COUNT_OFFSET = 18; + header.sequence_number = read_big_endian(packet, SEQUENCE_OFFSET); + header.message_count = read_big_endian(packet, COUNT_OFFSET); + + if (header.is_end_of_session()) { + return header; // Control packet: no message blocks, no sequence advance. + } + + // The sequence number names the first message in the packet; feed the count + // to the tracker so gaps in the multicast stream are detected. A heartbeat + // (count 0) still anchors the next expected sequence. + m_tracker.observe(header.session_view(), header.sequence_number, header.message_count); + + if (header.is_heartbeat()) { + return header; + } + + // After the header the datagram is a sequence of length-prefixed message + // blocks, which is exactly the framing the ordinary parser consumes. A + // malformed or truncated datagram is tolerated: the parser stops at the end + // of the buffer and we report through the callback what was decoded. + const std::span blocks = packet.subspan(HEADER_SIZE); + auto counting_callback = [this](const Message& message) { + ++m_messages_decoded; + if (m_callback) { + m_callback(message); + } + }; + // A truncated trailing block in a single datagram is non-fatal here; the gap + // will already have been surfaced by the sequence tracker. try_parse would + // avoid the exception, but it needs C++23's std::expected and this decoder + // must also build under C++20. + try { + m_parser.parse(blocks, counting_callback); + // NOLINTNEXTLINE(bugprone-empty-catch) + } catch (const std::runtime_error&) { + } + + return header; +} + +} // namespace itch::transport diff --git a/src/transport/pcap.cpp b/src/transport/pcap.cpp new file mode 100644 index 0000000..ac3cff1 --- /dev/null +++ b/src/transport/pcap.cpp @@ -0,0 +1,282 @@ +#include "itch/transport/pcap.hpp" + +#include +#include +#include +#include + +namespace itch::transport { + +namespace { + +// Capture-file record fields follow the file's own byte order; network protocol +// headers (Ethernet/IP/UDP) are always big-endian. These helpers read either. +template +auto read_le_or_be(std::span data, std::size_t offset, bool swapped) -> UintType { + UintType value {}; + std::memcpy(&value, data.data() + offset, sizeof(UintType)); + return swapped ? utils::swap_bytes(value) : value; +} + +template +auto read_net(std::span data, std::size_t offset) -> UintType { + UintType value {}; + std::memcpy(&value, data.data() + offset, sizeof(UintType)); + return utils::from_big_endian(value); +} + +// Link-layer types we know how to unwrap (subset of the tcpdump/pcap registry). +constexpr std::uint32_t LINKTYPE_ETHERNET = 1; +constexpr std::uint32_t LINKTYPE_RAW = 101; +constexpr std::uint32_t LINKTYPE_LINUX_SLL = 113; + +constexpr std::uint16_t ETHERTYPE_IPV4 = 0x0800; +constexpr std::uint16_t ETHERTYPE_IPV6 = 0x86DD; +constexpr std::uint16_t ETHERTYPE_VLAN = 0x8100; + +constexpr std::uint8_t IP_PROTOCOL_UDP = 17; + +constexpr std::size_t ETHERNET_HEADER_SIZE = 14; +constexpr std::size_t VLAN_TAG_SIZE = 4; +constexpr std::size_t LINUX_SLL_HEADER_SIZE = 16; +constexpr std::size_t IPV6_HEADER_SIZE = 40; +constexpr std::size_t UDP_HEADER_SIZE = 8; + +constexpr std::uint32_t PCAP_MAGIC = 0xA1B2C3D4; +constexpr std::uint32_t PCAP_MAGIC_SWAPPED = 0xD4C3B2A1; +constexpr std::uint32_t PCAP_MAGIC_NANO = 0xA1B23C4D; +constexpr std::uint32_t PCAP_MAGIC_NANO_SWP = 0x4D3CB2A1; +constexpr std::uint32_t PCAPNG_BLOCK_SHB = 0x0A0D0D0A; + +// Strips a frame's link-layer header down to the network (IP) layer, or +// returns std::nullopt if the frame is too short or its link type isn't one +// we know how to unwrap. +auto strip_link_layer(std::span frame, std::uint32_t link_type) + -> std::optional> { + if (link_type == LINKTYPE_ETHERNET) { + if (frame.size() < ETHERNET_HEADER_SIZE) { + return std::nullopt; + } + std::size_t header = ETHERNET_HEADER_SIZE; + auto ethertype = read_net(frame, 12); + while (ethertype == ETHERTYPE_VLAN && frame.size() >= header + VLAN_TAG_SIZE) { + ethertype = read_net(frame, header + 2); + header += VLAN_TAG_SIZE; + } + if (ethertype != ETHERTYPE_IPV4 && ethertype != ETHERTYPE_IPV6) { + return std::nullopt; + } + return frame.subspan(header); + } + if (link_type == LINKTYPE_LINUX_SLL) { + if (frame.size() < LINUX_SLL_HEADER_SIZE) { + return std::nullopt; + } + return frame.subspan(LINUX_SLL_HEADER_SIZE); + } + if (link_type != LINKTYPE_RAW) { + return std::nullopt; // Unsupported link layer. + } + return frame; +} + +} // namespace + +PcapReader::PcapReader(MessageCallback callback) : m_mold {std::move(callback)} {} + +auto PcapReader::read(std::span capture) -> bool { + if (capture.size() < sizeof(std::uint32_t)) { + return false; + } + std::uint32_t magic {}; + std::memcpy(&magic, capture.data(), sizeof(magic)); + + if (magic == PCAP_MAGIC || magic == PCAP_MAGIC_NANO) { + return read_classic_pcap(capture, /*swapped=*/false); + } + if (magic == PCAP_MAGIC_SWAPPED || magic == PCAP_MAGIC_NANO_SWP) { + return read_classic_pcap(capture, /*swapped=*/true); + } + if (magic == PCAPNG_BLOCK_SHB) { + return read_pcapng(capture); + } + return false; +} + +auto PcapReader::read_file(const std::string& path) -> bool { + std::ifstream file {path, std::ios::binary}; + if (!file) { + return false; + } + std::vector raw {std::istreambuf_iterator {file}, {}}; + std::vector bytes(raw.size()); + std::memcpy(bytes.data(), raw.data(), raw.size()); + return read(std::span {bytes}); +} + +auto PcapReader::read_classic_pcap(std::span capture, bool swapped) -> bool { + constexpr std::size_t GLOBAL_HEADER_SIZE = 24; + constexpr std::size_t NETWORK_OFFSET = 20; + constexpr std::size_t RECORD_HEADER_SIZE = 16; + constexpr std::size_t INCL_LEN_OFFSET = 8; + + if (capture.size() < GLOBAL_HEADER_SIZE) { + return false; + } + const auto link_type = read_le_or_be(capture, NETWORK_OFFSET, swapped); + + std::size_t offset = GLOBAL_HEADER_SIZE; + while (offset + RECORD_HEADER_SIZE <= capture.size()) { + const auto incl_len = + read_le_or_be(capture, offset + INCL_LEN_OFFSET, swapped); + offset += RECORD_HEADER_SIZE; + if (offset + incl_len > capture.size()) { + break; // Truncated final record. + } + handle_frame(capture.subspan(offset, incl_len), link_type); + offset += incl_len; + } + return true; +} + +auto PcapReader::read_pcapng(std::span capture) -> bool { + constexpr std::size_t BLOCK_HEADER_SIZE = 8; // Type + total length. + constexpr std::size_t BOM_OFFSET = 8; // Byte-order magic in the SHB. + constexpr std::uint32_t BOM_LITTLE = 0x1A2B3C4D; + constexpr std::uint32_t BLOCK_IDB = 0x00000001; + constexpr std::uint32_t BLOCK_SPB = 0x00000003; + constexpr std::uint32_t BLOCK_EPB = 0x00000006; + + bool swapped = false; + std::vector interface_link_types; + + std::size_t offset = 0; + while (offset + BLOCK_HEADER_SIZE <= capture.size()) { + std::uint32_t block_type {}; + std::memcpy(&block_type, capture.data() + offset, sizeof(block_type)); + // The SHB's byte-order magic tells us how to read this section. Until we + // have read it, block lengths are interpreted in host order, which is + // correct for the SHB type/length fields we need to bootstrap from. + const bool block_swapped = (block_type == PCAPNG_BLOCK_SHB) ? false : swapped; + + const auto total_length = + read_le_or_be(capture, offset + sizeof(std::uint32_t), block_swapped); + if (total_length < BLOCK_HEADER_SIZE + sizeof(std::uint32_t) || + offset + total_length > capture.size()) { + break; + } + + if (block_type == PCAPNG_BLOCK_SHB) { + const auto bom = read_net(capture, offset + BOM_OFFSET); + // If the byte-order magic reads as the canonical value in network + // (big-endian) interpretation, the file is big-endian; otherwise it + // is little-endian. Compare against both forms. + std::uint32_t bom_native {}; + std::memcpy(&bom_native, capture.data() + offset + BOM_OFFSET, sizeof(bom_native)); + swapped = (bom_native != BOM_LITTLE); + (void)bom; + interface_link_types.clear(); + } else if (block_type == BLOCK_IDB) { + // LinkType is the first 2-byte field of the IDB body. + const auto link_type = + read_le_or_be(capture, offset + BLOCK_HEADER_SIZE, swapped); + interface_link_types.push_back(link_type); + } else if (block_type == BLOCK_EPB) { + constexpr std::size_t EPB_IFACE_OFFSET = BLOCK_HEADER_SIZE; + constexpr std::size_t EPB_CAPLEN_OFFSET = BLOCK_HEADER_SIZE + 12; + constexpr std::size_t EPB_DATA_OFFSET = BLOCK_HEADER_SIZE + 20; + // The block must be large enough to hold its fixed header fields + // (interface id, two timestamp words, captured/packet length) plus + // the trailing repeated total-length word before those fields can + // be read; total_length alone (checked above) only guarantees the + // block fits in the buffer, not that it's this shape. + if (total_length >= EPB_DATA_OFFSET + sizeof(std::uint32_t)) { + const auto iface = + read_le_or_be(capture, offset + EPB_IFACE_OFFSET, swapped); + const auto cap_len = + read_le_or_be(capture, offset + EPB_CAPLEN_OFFSET, swapped); + if (offset + EPB_DATA_OFFSET + cap_len <= capture.size() && + iface < interface_link_types.size()) { + handle_frame( + capture.subspan(offset + EPB_DATA_OFFSET, cap_len), + interface_link_types[iface] + ); + } + } + } else if (block_type == BLOCK_SPB) { + constexpr std::size_t SPB_DATA_OFFSET = BLOCK_HEADER_SIZE + 4; + // The SPB has no captured length, so derive it from the block size. + if (total_length >= SPB_DATA_OFFSET + sizeof(std::uint32_t) && + !interface_link_types.empty()) { + const std::size_t data_len = total_length - SPB_DATA_OFFSET - sizeof(std::uint32_t); + handle_frame( + capture.subspan(offset + SPB_DATA_OFFSET, data_len), + interface_link_types.front() + ); + } + } + + offset += total_length; + } + return true; +} + +auto PcapReader::handle_frame(std::span frame, std::uint32_t link_type) -> void { + const auto network = strip_link_layer(frame, link_type); + if (!network.has_value()) { + return; + } + const auto payload = extract_udp_payload(*network); + if (!payload.has_value()) { + return; + } + ++m_udp_datagrams; + m_mold.decode_packet(*payload); +} + +auto PcapReader::extract_udp_payload(std::span network +) const -> std::optional> { + if (network.empty()) { + return std::nullopt; + } + + // Determine IP version and locate the UDP header. + const auto version = static_cast(network[0]) >> 4; + std::uint8_t protocol {0}; + std::size_t ip_header_len {0}; + if (version == 4) { + ip_header_len = static_cast(static_cast(network[0]) & 0x0F) * 4U; + if (network.size() < ip_header_len || ip_header_len < 20) { + return std::nullopt; + } + protocol = static_cast(network[9]); + } else if (version == 6) { + if (network.size() < IPV6_HEADER_SIZE) { + return std::nullopt; + } + protocol = static_cast(network[6]); + ip_header_len = IPV6_HEADER_SIZE; + } else { + return std::nullopt; + } + if (protocol != IP_PROTOCOL_UDP) { + return std::nullopt; + } + + const std::span udp = network.subspan(ip_header_len); + if (udp.size() < UDP_HEADER_SIZE) { + return std::nullopt; + } + const auto dst_port = read_net(udp, 2); + if (m_port_filter.has_value() && dst_port != *m_port_filter) { + return std::nullopt; + } + const auto udp_length = read_net(udp, 4); + std::size_t payload_len = udp.size() - UDP_HEADER_SIZE; + if (udp_length >= UDP_HEADER_SIZE) { + payload_len = std::min(payload_len, udp_length - UDP_HEADER_SIZE); + } + return udp.subspan(UDP_HEADER_SIZE, payload_len); +} + +} // namespace itch::transport diff --git a/src/transport/soupbintcp.cpp b/src/transport/soupbintcp.cpp new file mode 100644 index 0000000..dd059a8 --- /dev/null +++ b/src/transport/soupbintcp.cpp @@ -0,0 +1,159 @@ +#include "itch/transport/soupbintcp.hpp" + +#include +#include +#include +#include +#include +#include + +namespace itch::transport { + +namespace { + +constexpr std::size_t LENGTH_PREFIX_SIZE = 2; +constexpr std::size_t LOGIN_SESSION_SIZE = 10; +constexpr std::size_t LOGIN_SEQUENCE_SIZE = 20; +constexpr std::size_t MAX_WIRE_MESSAGE_LEN = 0xFFFF; + +// Trims trailing spaces from a fixed-width ASCII field. +auto trim_field(std::span field) -> std::string { + std::size_t length = field.size(); + while (length > 0 && static_cast(field[length - 1]) == ' ') { + --length; + } + std::string result(length, '\0'); + for (std::size_t index = 0; index < length; ++index) { + result[index] = static_cast(field[index]); + } + return result; +} + +// Parses the right-justified numeric sequence number field of Login Accepted. +auto parse_sequence_field(std::span field) -> std::uint64_t { + const std::string text = trim_field(field); + std::uint64_t value = 0; + const char* begin = text.data(); + // Skip any leading spaces left after trimming only trailing ones. + while (begin != text.data() + text.size() && *begin == ' ') { + ++begin; + } + std::from_chars(begin, text.data() + text.size(), value); + return value; +} + +} // namespace + +SoupBinDecoder::SoupBinDecoder(MessageCallback callback) : m_callback {std::move(callback)} {} + +auto SoupBinDecoder::set_event_callback(EventCallback callback) -> void { + m_event_callback = std::move(callback); +} + +auto SoupBinDecoder::feed(std::span bytes) -> void { + m_buffer.insert(m_buffer.end(), bytes.begin(), bytes.end()); + + std::size_t offset = 0; + while (m_buffer.size() - offset >= LENGTH_PREFIX_SIZE) { + std::uint16_t packet_length {}; + std::memcpy(&packet_length, m_buffer.data() + offset, LENGTH_PREFIX_SIZE); + packet_length = utils::from_big_endian(packet_length); + + if (packet_length == 0) { + offset += LENGTH_PREFIX_SIZE; // Defensive: skip a zero-length frame. + continue; + } + if (m_buffer.size() - offset < LENGTH_PREFIX_SIZE + packet_length) { + break; // The rest of this packet has not arrived yet. + } + + const auto type = + static_cast(static_cast(m_buffer[offset + LENGTH_PREFIX_SIZE]) + ); + const std::span payload { + m_buffer.data() + offset + LENGTH_PREFIX_SIZE + 1, + static_cast(packet_length) - 1 + }; + process_packet(type, payload); + offset += LENGTH_PREFIX_SIZE + packet_length; + } + + // Drop the bytes we have fully consumed, keeping any partial trailing packet. + if (offset > 0) { + m_buffer.erase(m_buffer.begin(), m_buffer.begin() + static_cast(offset)); + } +} + +auto SoupBinDecoder::process_packet(SoupBinPacketType type, std::span payload) + -> void { + switch (type) { + case SoupBinPacketType::login_accepted: { + if (payload.size() >= LOGIN_SESSION_SIZE + LOGIN_SEQUENCE_SIZE) { + m_session = trim_field(payload.subspan(0, LOGIN_SESSION_SIZE)); + m_next_sequence = + parse_sequence_field(payload.subspan(LOGIN_SESSION_SIZE, LOGIN_SEQUENCE_SIZE)); + if (m_next_sequence == 0) { + m_next_sequence = 1; + } + } + break; + } + case SoupBinPacketType::sequenced_data: { + m_tracker.observe(m_session, m_next_sequence, 1); + ++m_next_sequence; + decode_application_message(payload); + return; + } + case SoupBinPacketType::unsequenced_data: { + decode_application_message(payload); + return; + } + case SoupBinPacketType::debug: + case SoupBinPacketType::login_rejected: + case SoupBinPacketType::server_heartbeat: + case SoupBinPacketType::end_of_session: + case SoupBinPacketType::login_request: + case SoupBinPacketType::client_heartbeat: + case SoupBinPacketType::logout_request: + break; + } + + if (m_event_callback) { + m_event_callback(type, payload); + } +} + +auto SoupBinDecoder::decode_application_message(std::span payload) -> void { + if (payload.empty() || payload.size() > MAX_WIRE_MESSAGE_LEN) { + return; + } + + // A data packet carries one ITCH message without its own 2-byte length + // prefix; re-prefixing it lets the ordinary framing parser decode it. The + // length is written big-endian (network order) to match the wire framing. + std::vector framed; + framed.reserve(LENGTH_PREFIX_SIZE + payload.size()); + const auto length = static_cast(payload.size()); + const auto big_endian = utils::from_big_endian(length); + const auto len_bytes = std::bit_cast>(big_endian); + framed.push_back(len_bytes[0]); + framed.push_back(len_bytes[1]); + framed.insert(framed.end(), payload.begin(), payload.end()); + + auto counting_callback = [this](const Message& message) { + ++m_messages_decoded; + if (m_callback) { + m_callback(message); + } + }; + // A malformed single-message payload is skipped rather than aborting the + // whole stream. try_parse would avoid the exception, but it needs C++23's + // std::expected and this decoder must also build under C++20. + try { + m_parser.parse(std::span {framed}, counting_callback); + // NOLINTNEXTLINE(bugprone-empty-catch) + } catch (const std::runtime_error&) { + } +} + +} // namespace itch::transport diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d6bec98..fb4a925 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.23) +cmake_minimum_required(VERSION 3.25) find_package(GTest CONFIG QUIET) @@ -16,10 +16,22 @@ endif() add_executable( itch_tests test_parser.cpp + test_parser_edge.cpp test_messages.cpp test_order_book.cpp + test_order_book_extra.cpp + test_price_time.cpp + test_conformance.cpp + transport/test_transport.cpp + book/test_book.cpp + book/test_overlay.cpp + analytics/test_analytics.cpp + io/test_csv_sink.cpp + test_encoder.cpp ) +target_include_directories(itch_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries( itch_tests PRIVATE itch::itch diff --git a/tests/analytics/test_analytics.cpp b/tests/analytics/test_analytics.cpp new file mode 100644 index 0000000..81714f5 --- /dev/null +++ b/tests/analytics/test_analytics.cpp @@ -0,0 +1,173 @@ +#include + +#include +#include +#include + +#include "itch/analytics/auctions.hpp" +#include "itch/analytics/bars.hpp" +#include "itch/analytics/imbalance.hpp" +#include "itch/analytics/microstructure.hpp" +#include "itch/analytics/vwap.hpp" +#include "itch/book/l3_book.hpp" +#include "itch/messages.hpp" +#include "itch/tape.hpp" + +namespace { + +auto make_trade(std::uint64_t timestamp, std::uint32_t raw_price, std::uint64_t shares) + -> itch::Trade { + itch::Trade trade {}; + trade.timestamp = timestamp; + trade.price = itch::StandardPrice {raw_price}; + trade.shares = shares; + return trade; +} + +auto fill_stock(char (&dest)[itch::STOCK_LEN], std::string_view symbol) -> void { + std::memset(dest, ' ', itch::STOCK_LEN); + std::memcpy( + dest, symbol.data(), std::min(symbol.size(), static_cast(itch::STOCK_LEN)) + ); +} + +} // namespace + +TEST(Bars, TimeClockGroupsTradesIntoIntervals) { + std::vector bars; + itch::analytics::BarBuilder builder { + itch::analytics::TimeClock {10}, + [&](const itch::analytics::Bar& bar) { bars.push_back(bar); } + }; + + builder.add(make_trade(1, 100, 5)); // bucket 0 + builder.add(make_trade(5, 120, 3)); // bucket 0 + builder.add(make_trade(12, 90, 4)); // bucket 1 -> closes bucket 0 + builder.flush(); + + ASSERT_EQ(bars.size(), 2U); + EXPECT_EQ(bars[0].open.raw(), 100U); + EXPECT_EQ(bars[0].high.raw(), 120U); + EXPECT_EQ(bars[0].low.raw(), 100U); + EXPECT_EQ(bars[0].close.raw(), 120U); + EXPECT_EQ(bars[0].volume, 8U); + EXPECT_EQ(bars[0].trade_count, 2U); + EXPECT_EQ(bars[1].open.raw(), 90U); + EXPECT_EQ(bars[1].volume, 4U); +} + +TEST(Bars, VolumeClockClosesBarsOnVolumeThreshold) { + std::vector bars; + itch::analytics::BarBuilder builder { + itch::analytics::VolumeClock {100}, + [&](const itch::analytics::Bar& bar) { bars.push_back(bar); } + }; + builder.add(make_trade(1, 100, 60)); // cumulative 0 -> bucket 0 + builder.add(make_trade(2, 101, 50)); // cumulative 60 -> bucket 0 (now 110) + builder.add(make_trade(3, 102, 10)); // cumulative 110 -> bucket 1 -> closes bar 0 + builder.flush(); + ASSERT_EQ(bars.size(), 2U); + EXPECT_EQ(bars[0].volume, 110U); +} + +TEST(Vwap, ComputesVolumeWeightedAverage) { + itch::analytics::Vwap vwap; + vwap.add(itch::StandardPrice {100}, 10); + vwap.add(itch::StandardPrice {200}, 30); + // (100*10 + 200*30) / 40 = 175 raw -> /10000 = 0.0175 + EXPECT_DOUBLE_EQ(vwap.value(), 175.0 / 10000.0); + EXPECT_EQ(vwap.volume(), 40U); +} + +TEST(Twap, ComputesTimeWeightedAverage) { + itch::analytics::Twap twap; + twap.add(itch::StandardPrice {10000}, 0); // price 1.0 from t=0 + twap.add(itch::StandardPrice {30000}, 10); // 1.0 held for 10 units, then 3.0 + twap.add(itch::StandardPrice {30000}, 20); // 3.0 held for 10 units + // (1.0*10 + 3.0*10) / 20 = 2.0 + EXPECT_DOUBLE_EQ(twap.value(), 2.0); +} + +TEST(Microstructure, SpreadMidAndQueueImbalance) { + itch::book::Bbo bbo {}; + bbo.has_bid = true; + bbo.has_ask = true; + bbo.bid_price = itch::StandardPrice {1000000}; + bbo.ask_price = itch::StandardPrice {1000200}; + bbo.bid_shares = 300; + bbo.ask_shares = 100; + + EXPECT_NEAR(itch::analytics::spread(bbo), 0.02, 1e-9); + EXPECT_NEAR(itch::analytics::mid_price(bbo), 100.01, 1e-9); + EXPECT_DOUBLE_EQ(itch::analytics::queue_imbalance(bbo), 0.5); // (300-100)/400 +} + +TEST(Microstructure, OrderFlowImbalanceFollowsContModel) { + itch::book::Bbo previous {}; + previous.bid_price = itch::StandardPrice {1000000}; + previous.ask_price = itch::StandardPrice {1000200}; + previous.bid_shares = 100; + previous.ask_shares = 100; + + itch::book::Bbo current = previous; + current.bid_shares = 150; // size added at unchanged best bid -> +50 buy flow + EXPECT_DOUBLE_EQ(itch::analytics::order_flow_imbalance(previous, current), 50.0); +} + +TEST(Microstructure, DepthAtLevelSumsBestLevels) { + itch::book::L3Book book; + book.add_order(1, itch::book::Side::buy, 100, 1000000); + book.add_order(2, itch::book::Side::buy, 200, 999900); + book.add_order(3, itch::book::Side::buy, 300, 999800); + EXPECT_EQ(itch::analytics::depth_at_level(book, itch::book::Side::buy, 2), 300U); + EXPECT_EQ(itch::analytics::depth_at_level(book, itch::book::Side::buy, 0), 600U); +} + +TEST(Imbalance, SurfacesNoiiFields) { + itch::NOIIMessage msg {}; + msg.stock_locate = 5; + msg.timestamp = 42; + fill_stock(msg.stock, "AAPL"); + msg.paired_shares = 1000; + msg.imbalance_shares = 250; + msg.imbalance_direction = 'B'; + msg.current_reference_price = 1500000; + + const auto info = itch::analytics::make_imbalance_info(msg); + EXPECT_EQ(info.stock, "AAPL"); + EXPECT_EQ(info.paired_shares, 1000U); + EXPECT_EQ(info.imbalance_shares, 250U); + EXPECT_EQ(info.current_reference_price.raw(), 1500000U); + EXPECT_EQ(itch::analytics::imbalance_direction_name('B'), "Buy imbalance"); +} + +TEST(Auctions, ReconstructsCrossWithImbalanceContext) { + itch::analytics::AuctionTracker tracker; + std::vector auctions; + tracker.set_auction_callback([&](const itch::analytics::Auction& auction) { + auctions.push_back(auction); + }); + + itch::NOIIMessage noii {}; + noii.stock_locate = 7; + fill_stock(noii.stock, "MSFT"); + noii.paired_shares = 5000; + noii.imbalance_shares = 1200; + noii.imbalance_direction = 'S'; + tracker.process(itch::Message {noii}); + + itch::CrossTradeMessage cross {}; + cross.stock_locate = 7; + fill_stock(cross.stock, "MSFT"); + cross.shares = 6200; + cross.cross_price = 3000000; + cross.cross_type = 'O'; + tracker.process(itch::Message {cross}); + + ASSERT_EQ(auctions.size(), 1U); + EXPECT_EQ(auctions[0].cross_price.raw(), 3000000U); + EXPECT_EQ(auctions[0].cross_type, 'O'); + EXPECT_TRUE(auctions[0].had_imbalance); + EXPECT_EQ(auctions[0].imbalance_shares, 1200U); + EXPECT_EQ(itch::analytics::cross_type_name('O'), "Opening Cross"); +} diff --git a/tests/book/test_book.cpp b/tests/book/test_book.cpp new file mode 100644 index 0000000..f0cccb8 --- /dev/null +++ b/tests/book/test_book.cpp @@ -0,0 +1,173 @@ +#include + +#include +#include +#include + +#include "itch/book/book_manager.hpp" +#include "itch/book/l3_book.hpp" +#include "itch/messages.hpp" + +namespace { + +using itch::book::L3Book; +using itch::book::Side; + +// Fills an 8-byte ITCH stock field from a symbol, space padded. +auto fill_stock(char (&dest)[itch::STOCK_LEN], std::string_view symbol) -> void { + std::memset(dest, ' ', itch::STOCK_LEN); + std::memcpy( + dest, symbol.data(), std::min(symbol.size(), static_cast(itch::STOCK_LEN)) + ); +} + +auto make_add( + std::uint16_t locate, + std::uint64_t ref, + char side, + std::uint32_t shares, + std::string_view symbol, + std::uint32_t price +) -> itch::AddOrderMessage { + itch::AddOrderMessage msg {}; + msg.stock_locate = locate; + msg.order_reference_number = ref; + msg.buy_sell_indicator = side; + msg.shares = shares; + fill_stock(msg.stock, symbol); + msg.price = price; + return msg; +} + +} // namespace + +TEST(L3Book, TracksBestBidOfferAcrossLevels) { + L3Book book {"AAPL"}; + book.add_order(1, Side::buy, 100, 1500000); + book.add_order(2, Side::buy, 200, 1500100); // better bid + book.add_order(3, Side::sell, 50, 1500300); + book.add_order(4, Side::sell, 75, 1500200); // better ask + + const auto bbo = book.bbo(); + ASSERT_TRUE(bbo.has_bid); + ASSERT_TRUE(bbo.has_ask); + EXPECT_EQ(bbo.bid_price.raw(), 1500100U); + EXPECT_EQ(bbo.bid_shares, 200U); + EXPECT_EQ(bbo.ask_price.raw(), 1500200U); + EXPECT_EQ(bbo.ask_shares, 75U); +} + +TEST(L3Book, AggregatesSharesAndKeepsTimePriority) { + L3Book book; + book.add_order(1, Side::buy, 100, 1000000); + book.add_order(2, Side::buy, 150, 1000000); // same level, later in queue + const auto depth = book.depth(Side::buy); + ASSERT_EQ(depth.size(), 1U); + EXPECT_EQ(depth.front().shares, 250U); + EXPECT_EQ(depth.front().order_count, 2U); + + const auto orders = book.orders_at(Side::buy, 1000000); + ASSERT_EQ(orders.size(), 2U); + EXPECT_EQ(orders[0].reference_number, 1U); // FIFO order preserved + EXPECT_EQ(orders[1].reference_number, 2U); +} + +TEST(L3Book, ExecutePartialThenFullRemovesOrder) { + L3Book book; + book.add_order(1, Side::sell, 100, 2000000); + EXPECT_EQ(book.execute_order(1, 40), 40U); + EXPECT_EQ(book.bbo().ask_shares, 60U); + EXPECT_EQ(book.execute_order(1, 999), 60U); // clamps to remaining + EXPECT_FALSE(book.contains(1)); + EXPECT_FALSE(book.bbo().has_ask); +} + +TEST(L3Book, CancelReducesAndDeleteRemoves) { + L3Book book; + book.add_order(1, Side::buy, 500, 1000000); + book.reduce_order(1, 200); + EXPECT_EQ(book.bbo().bid_shares, 300U); + book.delete_order(1); + EXPECT_TRUE(book.empty()); +} + +TEST(L3Book, ReplaceMovesOrderToNewPriceAndReference) { + L3Book book; + book.add_order(1, Side::buy, 100, 1000000); + book.replace_order(1, 2, 80, 1000500); + EXPECT_FALSE(book.contains(1)); + ASSERT_TRUE(book.contains(2)); + EXPECT_EQ(book.bbo().bid_price.raw(), 1000500U); + EXPECT_EQ(book.bbo().bid_shares, 80U); +} + +TEST(L3Book, HandlesCrossedBookState) { + L3Book book; + book.add_order(1, Side::buy, 100, 1000500); // bid above + book.add_order(2, Side::sell, 100, 1000000); // ask below -> crossed + const auto bbo = book.bbo(); + EXPECT_GT(bbo.bid_price.raw(), bbo.ask_price.raw()); // book represents the crossed state +} + +TEST(BookManager, RoutesMessagesToPerSymbolBooks) { + itch::book::BookManager manager; + manager.process(itch::Message {make_add(1, 10, 'B', 100, "AAPL", 1500000)}); + manager.process(itch::Message {make_add(2, 11, 'S', 200, "MSFT", 3000000)}); + + EXPECT_EQ(manager.book_count(), 2U); + const auto* apple = manager.book_for_symbol("AAPL"); + ASSERT_NE(apple, nullptr); + EXPECT_EQ(apple->bbo().bid_shares, 100U); + EXPECT_EQ(manager.book(2)->symbol(), "MSFT"); +} + +TEST(BookManager, UniverseFilterTracksOnlySelectedSymbols) { + itch::book::BookManager manager; + manager.track_symbol("AAPL"); + manager.process(itch::Message {make_add(1, 10, 'B', 100, "AAPL", 1500000)}); + manager.process(itch::Message {make_add(2, 11, 'S', 200, "MSFT", 3000000)}); + + EXPECT_EQ(manager.book_count(), 1U); + EXPECT_NE(manager.book_for_symbol("AAPL"), nullptr); + EXPECT_EQ(manager.book_for_symbol("MSFT"), nullptr); +} + +TEST(BookManager, EmitsBboEventsOnlyWhenTopChanges) { + itch::book::BookManager manager; + int bbo_events = 0; + manager.set_bbo_callback([&](const L3Book&, const itch::book::Bbo&) { ++bbo_events; }); + + manager.process(itch::Message {make_add(1, 10, 'B', 100, "AAPL", 1500000)}); // new top + manager.process(itch::Message {make_add(1, 11, 'B', 100, "AAPL", 1400000)} + ); // worse, no change + EXPECT_EQ(bbo_events, 1); +} + +TEST(BookManager, ExtractsTradeTapeWithPrintableFlag) { + itch::book::BookManager manager; + std::vector tape; + manager.set_trade_callback([&](const itch::Trade& trade) { tape.push_back(trade); }); + + manager.process(itch::Message {make_add(1, 10, 'S', 100, "AAPL", 1500000)}); + + itch::OrderExecutedMessage exec {}; + exec.stock_locate = 1; + exec.order_reference_number = 10; + exec.executed_shares = 40; + exec.match_number = 999; + manager.process(itch::Message {exec}); + + itch::OrderExecutedWithPriceMessage exec_price {}; + exec_price.stock_locate = 1; + exec_price.order_reference_number = 10; + exec_price.executed_shares = 10; + exec_price.printable = 'N'; + exec_price.execution_price = 1499000; + manager.process(itch::Message {exec_price}); + + ASSERT_EQ(tape.size(), 2U); + EXPECT_EQ(tape[0].price.raw(), 1500000U); // E uses the resting order price + EXPECT_TRUE(tape[0].printable); + EXPECT_EQ(tape[1].price.raw(), 1499000U); // C uses the execution price + EXPECT_FALSE(tape[1].printable); // honours the printable flag +} diff --git a/tests/book/test_overlay.cpp b/tests/book/test_overlay.cpp new file mode 100644 index 0000000..808f681 --- /dev/null +++ b/tests/book/test_overlay.cpp @@ -0,0 +1,85 @@ +#include + +#include +#include + +#include "itch/messages.hpp" +#include "itch/overlay.hpp" +#include "itch/parser.hpp" +#include "transport/frame_builders.hpp" + +namespace { + +// Concatenates several length-prefixed frames into one buffer. +auto build_buffer(const std::vector>& payloads) -> std::vector { + std::vector buffer; + for (const auto& payload : payloads) { + const auto frame = itch::test::length_prefixed(payload); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + return buffer; +} + +} // namespace + +TEST(Overlay, ViewFieldsMatchEagerDecode) { + const auto payload = itch::test::add_order_payload(7, 42, 'B', 500, "AAPL", 1500000); + const auto buffer = build_buffer({payload}); + + // Eager decode for the reference values. + itch::Parser parser; + std::vector messages = parser.parse(std::span {buffer}); + ASSERT_EQ(messages.size(), 1U); + const auto& eager = std::get(messages.front()); + + // Lazy overlay decode of the same bytes. + itch::overlay::AddOrderView view {}; + std::uint64_t count = itch::overlay::for_each_message( + std::span {buffer}, + [&](const itch::overlay::MessageView& generic) { + view = itch::overlay::AddOrderView {generic.data(), generic.size()}; + } + ); + ASSERT_EQ(count, 1U); + + EXPECT_EQ(view.type(), 'A'); + EXPECT_EQ(view.stock_locate(), eager.stock_locate); + EXPECT_EQ(view.timestamp(), eager.timestamp); + EXPECT_EQ(view.order_reference_number(), eager.order_reference_number); + EXPECT_EQ(view.buy_sell_indicator(), eager.buy_sell_indicator); + EXPECT_EQ(view.shares(), eager.shares); + EXPECT_EQ(view.price(), eager.price); + EXPECT_EQ(itch::to_string(view.stock().data(), view.stock().size()), "AAPL"); +} + +TEST(Overlay, FramesMultipleMessagesAndSkipsUnknownTypes) { + std::vector> payloads = { + itch::test::add_order_payload(1, 1, 'B', 100, "AAA", 1000000), + itch::test::system_event_payload(5000, 'O'), + itch::test::add_order_payload(1, 2, 'S', 200, "AAA", 1000100), + }; + const auto buffer = build_buffer(payloads); + + std::uint64_t adds = 0; + const auto total = itch::overlay::for_each_message( + std::span {buffer}, + [&](const itch::overlay::MessageView& view) { + if (view.type() == 'A') { + ++adds; + } + } + ); + EXPECT_EQ(total, 3U); + EXPECT_EQ(adds, 2U); +} + +TEST(Overlay, UndersizedFrameIsSkipped) { + // A frame claiming type 'A' but only 5 bytes long must not be delivered. + std::vector payload = { + std::byte {'A'}, std::byte {0}, std::byte {0}, std::byte {0}, std::byte {0} + }; + const auto buffer = build_buffer({payload}); + const auto total = + itch::overlay::for_each_message(std::span {buffer}, [](const auto&) {}); + EXPECT_EQ(total, 0U); +} diff --git a/tests/io/test_csv_sink.cpp b/tests/io/test_csv_sink.cpp new file mode 100644 index 0000000..d7d51e3 --- /dev/null +++ b/tests/io/test_csv_sink.cpp @@ -0,0 +1,46 @@ +#include + +#include +#include +#include +#include + +#include "itch/io/csv_sink.hpp" +#include "itch/parser.hpp" +#include "transport/frame_builders.hpp" + +namespace { + +auto build_buffer(const std::vector>& payloads) -> std::vector { + std::vector buffer; + for (const auto& payload : payloads) { + const auto frame = itch::test::length_prefixed(payload); + buffer.insert(buffer.end(), frame.begin(), frame.end()); + } + return buffer; +} + +} // namespace + +TEST(CsvSink, WritesHeaderAndNormalizedRows) { + const auto buffer = build_buffer({ + itch::test::system_event_payload(1000, 'O'), + itch::test::add_order_payload(7, 42, 'B', 500, "AAPL", 1500000), + }); + + std::ostringstream out; + itch::io::CsvSink sink {out}; + itch::Parser parser; + parser.parse(std::span {buffer}, [&](const itch::Message& msg) { + sink.write(msg); + }); + sink.flush(); + + const std::string text = out.str(); + EXPECT_NE(text.find("message_type,timestamp,stock_locate"), std::string::npos); + // System event row carries its event code in the extra column. + EXPECT_NE(text.find("event=O"), std::string::npos); + // Add order row carries symbol, side, shares, and a 4-decimal price. + EXPECT_NE(text.find("AAPL,42,B,500,150.0000"), std::string::npos); + EXPECT_EQ(sink.rows_written(), 2U); +} diff --git a/tests/test_conformance.cpp b/tests/test_conformance.cpp new file mode 100644 index 0000000..ae8f6cd --- /dev/null +++ b/tests/test_conformance.cpp @@ -0,0 +1,134 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include "itch/parser.hpp" + +// A golden/conformance test: a deterministic, hand-built ITCH stream with a known +// composition is parsed, and the per-type message counts and spot-checked field +// values are asserted against the known-good fixture. This gives an end-to-end +// "known input produces known output" check without committing a multi-gigabyte +// binary sample to the repository. +namespace { + +class StreamBuilder { + public: + auto system_event(char event_code) -> void { + std::vector payload; + payload.push_back(static_cast('S')); + header(payload, ++m_seq); + payload.push_back(static_cast(event_code)); + frame(payload); + } + + auto add_order(std::uint64_t order_ref, char side, std::uint32_t shares, std::uint32_t price) + -> void { + std::vector payload; + payload.push_back(static_cast('A')); + header(payload, ++m_seq); + u64(payload, order_ref); + payload.push_back(static_cast(side)); + u32(payload, shares); + for (int idx = 0; idx < 8; ++idx) { + payload.push_back(static_cast("AAPL "[idx])); + } + u32(payload, price); + frame(payload); + } + + auto order_executed(std::uint64_t order_ref, std::uint32_t executed, std::uint64_t match) + -> void { + std::vector payload; + payload.push_back(static_cast('E')); + header(payload, ++m_seq); + u64(payload, order_ref); + u32(payload, executed); + u64(payload, match); + frame(payload); + } + + [[nodiscard]] auto span() const -> std::span { + return std::as_bytes(std::span {m_bytes}); + } + + private: + static auto u16(std::vector& out, std::uint16_t value) -> void { + out.push_back(static_cast(value >> 8)); + out.push_back(static_cast(value & 0xFF)); + } + static auto u32(std::vector& out, std::uint32_t value) -> void { + for (int shift = 24; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + static auto u48(std::vector& out, std::uint64_t value) -> void { + for (int shift = 40; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + static auto u64(std::vector& out, std::uint64_t value) -> void { + for (int shift = 56; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + static auto header(std::vector& out, std::uint16_t seq) -> void { + u16(out, seq); // stock_locate + u16(out, 0); // tracking_number + u48(out, seq); // timestamp + } + auto frame(const std::vector& payload) -> void { + const auto length = static_cast(payload.size()); + m_bytes.push_back(static_cast(length >> 8)); + m_bytes.push_back(static_cast(length & 0xFF)); + m_bytes.insert(m_bytes.end(), payload.begin(), payload.end()); + } + + std::vector m_bytes; + std::uint16_t m_seq {0}; +}; + +} // namespace + +TEST(Conformance, KnownStreamProducesKnownCountsAndFields) { + StreamBuilder builder; + // Known composition: 3 system events, 5 add orders, 2 executions. + builder.system_event('O'); + builder.add_order(10, 'B', 100, 5000); + builder.add_order(11, 'B', 200, 4900); + builder.system_event('S'); + builder.add_order(12, 'S', 150, 5100); + builder.add_order(13, 'S', 50, 5200); + builder.order_executed(10, 40, 1); + builder.add_order(14, 'B', 300, 4800); + builder.order_executed(12, 150, 2); + builder.system_event('C'); + + itch::Parser parser; + auto messages = parser.parse(builder.span()); + + std::map counts; + for (const auto& message : messages) { + const char type = std::visit([](auto&& msg) { return msg.message_type; }, message); + ++counts[type]; + } + + EXPECT_EQ(messages.size(), 10u); + EXPECT_EQ(counts['S'], 3); + EXPECT_EQ(counts['A'], 5); + EXPECT_EQ(counts['E'], 2); + EXPECT_EQ(parser.unknown_message_count(), 0u); + EXPECT_EQ(parser.malformed_message_count(), 0u); + + // Spot-check fields of the first add order. + const auto& first_add = std::get(messages[1]); + EXPECT_EQ(first_add.order_reference_number, 10u); + EXPECT_EQ(first_add.buy_sell_indicator, 'B'); + EXPECT_EQ(first_add.shares, 100u); + EXPECT_EQ(first_add.price, 5000u); + EXPECT_EQ(itch::to_string(first_add.stock, 8), "AAPL"); +} diff --git a/tests/test_encoder.cpp b/tests/test_encoder.cpp new file mode 100644 index 0000000..b9dd532 --- /dev/null +++ b/tests/test_encoder.cpp @@ -0,0 +1,125 @@ +#include + +#include +#include +#include +#include + +#include "itch/encoder.hpp" +#include "itch/messages.hpp" +#include "itch/parser.hpp" +#include "itch/replay.hpp" +#include "itch/venue.hpp" + +namespace { + +auto fill_stock(char (&dest)[itch::STOCK_LEN], std::string_view symbol) -> void { + std::memset(dest, ' ', itch::STOCK_LEN); + std::memcpy( + dest, symbol.data(), std::min(symbol.size(), static_cast(itch::STOCK_LEN)) + ); +} + +// Round-trips a message through encode -> parse and returns the decoded variant. +auto round_trip(const itch::Message& message) -> itch::Message { + const std::vector frame = itch::encode_frame(message); + itch::Parser parser; + std::vector out = parser.parse(std::span {frame}); + EXPECT_EQ(out.size(), 1U); + return out.front(); +} + +} // namespace + +TEST(Encoder, AddOrderRoundTrips) { + itch::AddOrderMessage original {}; + original.stock_locate = 7; + original.tracking_number = 3; + original.timestamp = 0x0000123456789ABCULL & 0x0000FFFFFFFFFFFFULL; // fits 48 bits + original.order_reference_number = 987654321; + original.buy_sell_indicator = 'B'; + original.shares = 500; + fill_stock(original.stock, "AAPL"); + original.price = 1500000; + + const auto decoded = std::get(round_trip(itch::Message {original})); + EXPECT_EQ(decoded.stock_locate, original.stock_locate); + EXPECT_EQ(decoded.tracking_number, original.tracking_number); + EXPECT_EQ(decoded.timestamp, original.timestamp); + EXPECT_EQ(decoded.order_reference_number, original.order_reference_number); + EXPECT_EQ(decoded.buy_sell_indicator, 'B'); + EXPECT_EQ(decoded.shares, 500U); + EXPECT_EQ(itch::to_string(decoded.stock, itch::STOCK_LEN), "AAPL"); + EXPECT_EQ(decoded.price, 1500000U); +} + +TEST(Encoder, NoiiRoundTripsAllFields) { + itch::NOIIMessage original {}; + original.stock_locate = 11; + original.timestamp = 42; + original.paired_shares = 100000; + original.imbalance_shares = 2500; + original.imbalance_direction = 'B'; + fill_stock(original.stock, "MSFT"); + original.far_price = 3000000; + original.near_price = 3000100; + original.current_reference_price = 3000050; + original.cross_type = 'O'; + original.price_variation_indicator = 'L'; + + const auto decoded = std::get(round_trip(itch::Message {original})); + EXPECT_EQ(decoded.paired_shares, 100000U); + EXPECT_EQ(decoded.imbalance_shares, 2500U); + EXPECT_EQ(decoded.imbalance_direction, 'B'); + EXPECT_EQ(itch::to_string(decoded.stock, itch::STOCK_LEN), "MSFT"); + EXPECT_EQ(decoded.near_price, 3000100U); + EXPECT_EQ(decoded.cross_type, 'O'); +} + +TEST(Encoder, FrameCarriesLengthPrefix) { + itch::SystemEventMessage event {}; + event.event_code = 'O'; + const auto frame = itch::encode_frame(itch::Message {event}); + const auto body = itch::encode_message(itch::Message {event}); + // The 2-byte length prefix should equal the body size. + const auto length = + (static_cast(frame[0]) << 8) | static_cast(frame[1]); + EXPECT_EQ(length, body.size()); + EXPECT_EQ(frame.size(), body.size() + 2); +} + +TEST(Replay, DeliversAllMessagesAndScalesTiming) { + // Two messages 100 ms apart in feed time; at 100x speed the wall gap is ~1 ms. + itch::AddOrderMessage first {}; + first.stock_locate = 1; + first.timestamp = 0; + fill_stock(first.stock, "AAA"); + itch::AddOrderMessage second = first; + second.timestamp = 100'000'000; // 100 ms later + + std::vector stream; + for (const auto& msg : {first, second}) { + const auto frame = itch::encode_frame(itch::Message {msg}); + stream.insert(stream.end(), frame.begin(), frame.end()); + } + + itch::ReplayEngine engine {100.0}; + int count = 0; + const auto start = std::chrono::steady_clock::now(); + const auto replayed = + engine.replay(std::span {stream}, [&](const itch::Message&) { ++count; }); + const auto elapsed = std::chrono::steady_clock::now() - start; + + EXPECT_EQ(replayed, 2U); + EXPECT_EQ(count, 2); + // 100 ms / 100x = ~1 ms expected; assert it paced at least a little but not + // anywhere near the full 100 ms (generous bounds to avoid flakiness). + EXPECT_LT(elapsed, std::chrono::milliseconds {80}); +} + +TEST(Venue, Nasdaq50PolicyEnumeratesMessageTypes) { + int count = 0; + itch::venue::Nasdaq50::for_each_message_type([&count](char) { ++count; }); + EXPECT_EQ(count, 23); // The full ITCH 5.0 message set. + EXPECT_EQ(itch::venue::Nasdaq50::name(), "NASDAQ TotalView-ITCH 5.0"); +} diff --git a/tests/test_order_book_extra.cpp b/tests/test_order_book_extra.cpp new file mode 100644 index 0000000..604fd7b --- /dev/null +++ b/tests/test_order_book_extra.cpp @@ -0,0 +1,98 @@ +#include + +#include + +#include "itch/messages.hpp" +#include "itch/order_book.hpp" + +namespace { + +auto make_add(std::uint64_t ref, char side, std::uint32_t shares, std::uint32_t price) + -> itch::AddOrderMessage { + itch::AddOrderMessage msg {}; + std::memcpy(msg.stock, "ZVZZT ", 8); + msg.order_reference_number = ref; + msg.buy_sell_indicator = side; + msg.shares = shares; + msg.price = price; + return msg; +} + +} // namespace + +class OrderBookExtra : public ::testing::Test { + protected: + OrderBookExtra() : book("ZVZZT") {} + itch::LimitOrderBook book; +}; + +// A crossed book (best bid >= best ask) can legitimately occur in a raw feed. +// The reconstruction must store both sides without matching or losing orders. +TEST_F(OrderBookExtra, CrossedBookKeepsBothSides) { + book.process(make_add(1, 'B', 100, 5100)); // Aggressive bid. + book.process(make_add(2, 'S', 100, 5000)); // Aggressive ask, below the bid. + + ASSERT_EQ(book.get_bids().size(), 1u); + ASSERT_EQ(book.get_asks().size(), 1u); + + const std::uint32_t best_bid = book.get_bids().begin()->first; + const std::uint32_t best_ask = book.get_asks().begin()->first; + EXPECT_EQ(best_bid, 5100u); + EXPECT_EQ(best_ask, 5000u); + EXPECT_GT(best_bid, best_ask); // The book is crossed, and that is preserved. +} + +TEST_F(OrderBookExtra, BestBidIsHighestBestAskIsLowest) { + book.process(make_add(1, 'B', 100, 4900)); + book.process(make_add(2, 'B', 100, 5000)); + book.process(make_add(3, 'B', 100, 4800)); + book.process(make_add(4, 'S', 100, 5300)); + book.process(make_add(5, 'S', 100, 5200)); + book.process(make_add(6, 'S', 100, 5400)); + + EXPECT_EQ(book.get_bids().begin()->first, 5000u); // Highest bid first. + EXPECT_EQ(book.get_asks().begin()->first, 5200u); // Lowest ask first. +} + +// Walk a single order through its full lifecycle across message types. +TEST_F(OrderBookExtra, FullLifecycleAddExecuteReplaceCancelDelete) { + book.process(make_add(100, 'B', 1000, 5000)); + ASSERT_EQ(book.get_bids().at(5000).total_shares, 1000u); + + // Partial execution. + itch::OrderExecutedMessage exec {}; + exec.order_reference_number = 100; + exec.executed_shares = 250; + book.process(exec); + EXPECT_EQ(book.get_bids().at(5000).total_shares, 750u); + + // Execution with price (partial). + itch::OrderExecutedWithPriceMessage exec_price {}; + exec_price.order_reference_number = 100; + exec_price.executed_shares = 250; + exec_price.execution_price = 5001; + book.process(exec_price); + EXPECT_EQ(book.get_bids().at(5000).total_shares, 500u); + + // Replace the remainder to a new price level. + itch::OrderReplaceMessage replace {}; + replace.original_order_reference_number = 100; + replace.new_order_reference_number = 101; + replace.shares = 500; + replace.price = 5050; + book.process(replace); + EXPECT_EQ(book.get_bids().count(5000), 0u); + EXPECT_EQ(book.get_bids().at(5050).total_shares, 500u); + + // Partial cancel, then delete the rest. + itch::OrderCancelMessage cancel {}; + cancel.order_reference_number = 101; + cancel.cancelled_shares = 200; + book.process(cancel); + EXPECT_EQ(book.get_bids().at(5050).total_shares, 300u); + + itch::OrderDeleteMessage del {}; + del.order_reference_number = 101; + book.process(del); + EXPECT_TRUE(book.get_bids().empty()); +} diff --git a/tests/test_parser.cpp b/tests/test_parser.cpp index 5d74725..922834d 100644 --- a/tests/test_parser.cpp +++ b/tests/test_parser.cpp @@ -89,7 +89,7 @@ class ParserTest : public ::testing::Test { }; TEST_F(ParserTest, SingleValidSystemEventMessage) { - itch::SystemEventMessage msg_to_pack{}; + itch::SystemEventMessage msg_to_pack {}; // NOLINTBEGIN msg_to_pack.stock_locate = 1; msg_to_pack.tracking_number = 2; @@ -115,13 +115,13 @@ TEST_F(ParserTest, SingleValidSystemEventMessage) { } TEST_F(ParserTest, MultipleValidMessages) { - itch::SystemEventMessage msg1_to_pack{}; + itch::SystemEventMessage msg1_to_pack {}; // NOLINTBEGIN msg1_to_pack.stock_locate = 1; msg1_to_pack.timestamp = 3; msg1_to_pack.event_code = 'O'; - itch::AddOrderMessage msg2_to_pack{}; + itch::AddOrderMessage msg2_to_pack {}; msg2_to_pack.order_reference_number = 12345; msg2_to_pack.buy_sell_indicator = 'B'; msg2_to_pack.shares = 100; @@ -156,7 +156,7 @@ TEST_F(ParserTest, MultipleValidMessages) { TEST_F(ParserTest, ThrowsOnIncompletePayload) { // Create a valid message, then truncate it - std::string data = create_test_buffer(itch::SystemEventMessage{}); + std::string data = create_test_buffer(itch::SystemEventMessage {}); data.pop_back(); // Make the payload incomplete std::stringstream data_stream(data); @@ -180,8 +180,8 @@ TEST_F(ParserTest, HandlesEmptyStream) { } TEST_F(ParserTest, CallbackBasedParsing) { - std::string data = create_test_buffer(itch::SystemEventMessage{}) + - create_test_buffer(itch::AddOrderMessage{}); + std::string data = create_test_buffer(itch::SystemEventMessage {}) + + create_test_buffer(itch::AddOrderMessage {}); std::stringstream data_stream(data); size_t message_count = 0; @@ -199,9 +199,9 @@ TEST_F(ParserTest, CallbackBasedParsing) { } TEST_F(ParserTest, FilteredParsing) { - std::string data = create_test_buffer(itch::SystemEventMessage{}) + - create_test_buffer(itch::StockDirectoryMessage{}) + - create_test_buffer(itch::AddOrderMessage{}); + std::string data = create_test_buffer(itch::SystemEventMessage {}) + + create_test_buffer(itch::StockDirectoryMessage {}) + + create_test_buffer(itch::AddOrderMessage {}); std::stringstream data_stream(data); auto messages = parser.parse(data_stream, {'R'}); @@ -212,9 +212,9 @@ TEST_F(ParserTest, FilteredParsing) { TEST_F(ParserTest, SkipsZeroLengthMessage) { const std::array zero_len_msg = {'\x00', '\x00'}; - std::string data = create_test_buffer(itch::SystemEventMessage{}) + + std::string data = create_test_buffer(itch::SystemEventMessage {}) + std::string(zero_len_msg.begin(), zero_len_msg.end()) + - create_test_buffer(itch::SystemEventMessage{}); + create_test_buffer(itch::SystemEventMessage {}); std::stringstream data_stream(data); @@ -223,14 +223,14 @@ TEST_F(ParserTest, SkipsZeroLengthMessage) { } TEST_F(ParserTest, IgnoresTrailingGarbageData) { - std::string data = create_test_buffer(itch::SystemEventMessage{}) + "garbage"; + std::string data = create_test_buffer(itch::SystemEventMessage {}) + "garbage"; std::stringstream data_stream(data); // It throws because the "garbage" is interpreted as an incomplete header EXPECT_THROW(parser.parse(data_stream), std::runtime_error); // To test ignoring trailing data, ensure it's smaller than a header - std::string data2 = create_test_buffer(itch::SystemEventMessage{}) + "g"; + std::string data2 = create_test_buffer(itch::SystemEventMessage {}) + "g"; std::stringstream ss2(data2); EXPECT_THROW(parser.parse(ss2), std::runtime_error); @@ -239,8 +239,8 @@ TEST_F(ParserTest, IgnoresTrailingGarbageData) { } TEST_F(ParserTest, HandlesStreamEndingExactlyOnBoundary) { - std::string data = create_test_buffer(itch::SystemEventMessage{}) + - create_test_buffer(itch::AddOrderMessage{}); + std::string data = create_test_buffer(itch::SystemEventMessage {}) + + create_test_buffer(itch::AddOrderMessage {}); std::stringstream data_stream(data); // Should parse cleanly with no exceptions diff --git a/tests/test_parser_edge.cpp b/tests/test_parser_edge.cpp new file mode 100644 index 0000000..7aa4e8f --- /dev/null +++ b/tests/test_parser_edge.cpp @@ -0,0 +1,216 @@ +#include + +#include +#include +#include +#include + +#include "itch/parser.hpp" + +namespace { + +// Minimal big-endian wire-frame writer mirroring the ITCH layout the parser +// expects: a 2-byte big-endian length prefix followed by the payload, whose +// header is type(1) + stock_locate(2) + tracking_number(2) + 48-bit timestamp. +class FrameWriter { + public: + auto add_system_event( + std::uint16_t locate, std::uint16_t tracking, std::uint64_t timestamp, char event_code + ) -> void { + std::vector payload; + payload.push_back(static_cast('S')); + append_u16(payload, locate); + append_u16(payload, tracking); + append_u48(payload, timestamp); + payload.push_back(static_cast(event_code)); + append_frame(payload); + } + + auto add_order( + std::uint16_t locate, + std::uint64_t order_ref, + char side, + std::uint32_t shares, + const char (&stock)[9], + std::uint32_t price + ) -> void { + std::vector payload; + payload.push_back(static_cast('A')); + append_u16(payload, locate); + append_u16(payload, 0); + append_u48(payload, 0); + append_u64(payload, order_ref); + payload.push_back(static_cast(side)); + append_u32(payload, shares); + for (int idx = 0; idx < 8; ++idx) { + payload.push_back(static_cast(stock[idx])); + } + append_u32(payload, price); + append_frame(payload); + } + + // Writes a frame with a caller-chosen (possibly wrong) declared length. + auto add_raw_frame(std::uint16_t declared_length, const std::vector& payload) + -> void { + m_bytes.push_back(static_cast(declared_length >> 8)); + m_bytes.push_back(static_cast(declared_length & 0xFF)); + m_bytes.insert(m_bytes.end(), payload.begin(), payload.end()); + } + + [[nodiscard]] auto bytes() const -> const std::vector& { return m_bytes; } + + [[nodiscard]] auto as_byte_span() const -> std::span { + return std::as_bytes(std::span {m_bytes}); + } + + private: + static auto append_u16(std::vector& out, std::uint16_t value) -> void { + out.push_back(static_cast(value >> 8)); + out.push_back(static_cast(value & 0xFF)); + } + static auto append_u32(std::vector& out, std::uint32_t value) -> void { + for (int shift = 24; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + static auto append_u48(std::vector& out, std::uint64_t value) -> void { + for (int shift = 40; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + static auto append_u64(std::vector& out, std::uint64_t value) -> void { + for (int shift = 56; shift >= 0; shift -= 8) { + out.push_back(static_cast((value >> shift) & 0xFF)); + } + } + auto append_frame(const std::vector& payload) -> void { + add_raw_frame(static_cast(payload.size()), payload); + } + + std::vector m_bytes; +}; + +constexpr char STOCK_AAPL[9] = "AAPL "; + +} // namespace + +TEST(ParserEdge, EndiannessRoundTrip) { + EXPECT_EQ(itch::utils::swap_bytes(0x1234), 0x3412); + EXPECT_EQ(itch::utils::swap_bytes(0x11223344), 0x44332211u); + EXPECT_EQ(itch::utils::swap_bytes(0x0102030405060708ull), 0x0807060504030201ull); + + // Swapping twice is the identity. + const std::uint32_t value = 0xDEADBEEF; + EXPECT_EQ(itch::utils::swap_bytes(itch::utils::swap_bytes(value)), value); + + // A single byte is unchanged. + EXPECT_EQ(itch::utils::swap_bytes(0x7F), 0x7F); +} + +TEST(ParserEdge, SpanOverloadParsesIdenticallyToPointer) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + writer.add_order(1, 42, 'B', 100, STOCK_AAPL, 5000); + + itch::Parser parser; + auto via_span = parser.parse(writer.as_byte_span()); + ASSERT_EQ(via_span.size(), 2u); + EXPECT_TRUE(std::holds_alternative(via_span[0])); + EXPECT_TRUE(std::holds_alternative(via_span[1])); +} + +TEST(ParserEdge, ZeroLengthFramesAreSkipped) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + writer.add_raw_frame(0, {}); // Zero-length padding frame. + writer.add_system_event(4, 5, 6, 'C'); + + itch::Parser parser; + auto messages = parser.parse(writer.as_byte_span()); + EXPECT_EQ(messages.size(), 2u); +} + +TEST(ParserEdge, UnknownTypeIsSkippedAndCounted) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + // A well-formed-looking frame whose type byte '?' is not a known message. + writer.add_raw_frame(4, {static_cast('?'), 0, 0, 0}); + writer.add_system_event(4, 5, 6, 'C'); + + itch::Parser parser; + int seen = 0; + parser.parse(writer.as_byte_span(), [&](const itch::Message&) { ++seen; }); + + EXPECT_EQ(seen, 2); + EXPECT_EQ(parser.unknown_message_count(), 1u); + EXPECT_EQ(parser.malformed_message_count(), 0u); +} + +TEST(ParserEdge, UndersizedFrameIsRejectedNotDecodedIntoNextFrame) { + FrameWriter writer; + // A frame claiming type 'S' (needs 12 bytes) but only declaring 5: it must be + // rejected as malformed rather than reading into the following frame. + writer.add_raw_frame(5, {static_cast('S'), 0, 0, 0, 0}); + writer.add_system_event(7, 8, 9, 'C'); + + itch::Parser parser; + auto messages = parser.parse(writer.as_byte_span()); + + ASSERT_EQ(messages.size(), 1u); + ASSERT_TRUE(std::holds_alternative(messages[0])); + EXPECT_EQ(std::get(messages[0]).stock_locate, 7); + EXPECT_EQ(parser.malformed_message_count(), 1u); +} + +TEST(ParserEdge, ErrorCallbackReceivesEachProblem) { + FrameWriter writer; + writer.add_raw_frame(4, {static_cast('?'), 0, 0, 0}); // unknown + writer.add_raw_frame(5, {static_cast('S'), 0, 0, 0, 0}); // undersized + + itch::Parser parser; + std::vector errors; + parser.set_error_callback([&](itch::ParseError error, char) { errors.push_back(error); }); + parser.parse(writer.as_byte_span(), [](const itch::Message&) {}); + + ASSERT_EQ(errors.size(), 2u); + EXPECT_EQ(errors[0], itch::ParseError::unknown_type); + EXPECT_EQ(errors[1], itch::ParseError::size_mismatch); +} + +TEST(ParserEdge, ThrowsOnTruncatedPayload) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + // Declare a 12-byte frame but provide only 4 payload bytes. + writer.add_raw_frame(12, {static_cast('S'), 0, 0, 0}); + + itch::Parser parser; + EXPECT_THROW(parser.parse(writer.as_byte_span()), std::runtime_error); +} + +#if defined(__cpp_lib_expected) +TEST(ParserEdge, TryParseReturnsTruncatedInsteadOfThrowing) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + writer.add_raw_frame(12, {static_cast('S'), 0, 0, 0}); // truncated + + itch::Parser parser; + int seen = 0; + auto result = parser.try_parse(writer.as_byte_span(), [&](const itch::Message&) { ++seen; }); + + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), itch::ParseError::truncated); + EXPECT_EQ(seen, 1); // The first, complete frame was still delivered. +} + +TEST(ParserEdge, TryParseSucceedsOnWellFormedInput) { + FrameWriter writer; + writer.add_system_event(1, 2, 3, 'O'); + writer.add_order(1, 42, 'B', 100, STOCK_AAPL, 5000); + + itch::Parser parser; + auto result = parser.try_parse(writer.as_byte_span()); + + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->size(), 2u); +} +#endif diff --git a/tests/test_price_time.cpp b/tests/test_price_time.cpp new file mode 100644 index 0000000..e8fe537 --- /dev/null +++ b/tests/test_price_time.cpp @@ -0,0 +1,63 @@ +#include + +#include +#include +#include + +#include "itch/price.hpp" +#include "itch/time.hpp" + +TEST(PriceTest, StandardScaleUsesFourDecimals) { + const auto price = itch::make_price(1500000); // 150.0000 + EXPECT_EQ(price.raw(), 1500000u); + EXPECT_EQ(itch::StandardPrice::divisor(), 10000u); + EXPECT_DOUBLE_EQ(price.to_double(), 150.0); + EXPECT_EQ(price.to_string(), "150.0000"); +} + +TEST(PriceTest, MwcbScaleUsesEightDecimals) { + const auto price = itch::make_mwcb_price(15000000000ull); // 150.00000000 + EXPECT_EQ(itch::MwcbPrice::divisor(), 100000000ull); + EXPECT_DOUBLE_EQ(price.to_double(), 150.0); + EXPECT_EQ(price.to_string(), "150.00000000"); +} + +TEST(PriceTest, TheTwoScalesAreDistinctTypes) { + // The standard and MWCB prices are different types, so the 4-vs-8 decimal + // mix-up cannot compile. This is a documentation-as-test of that intent. + static_assert(!std::is_same_v); + EXPECT_EQ(itch::StandardPrice::decimals, 4u); + EXPECT_EQ(itch::MwcbPrice::decimals, 8u); +} + +TEST(PriceTest, ComparesByRawValue) { + EXPECT_TRUE(itch::make_price(100) < itch::make_price(200)); + EXPECT_TRUE(itch::make_price(200) == itch::make_price(200)); +} + +TEST(PriceTest, FormatterRendersThroughStdFormat) { + EXPECT_EQ(std::format("{:.4f}", itch::make_price(1500000)), "150.0000"); +} + +TEST(TimeTest, CombinesSessionDateAndTimestamp) { + using namespace std::chrono; + // 09:30:00.000000000 = 34200 seconds past midnight. + constexpr std::uint64_t nanos = 34200ull * 1'000'000'000ull; + const auto point = itch::to_time_point(2020y / January / 30, nanos); + + const auto expected = sys_days {2020y / January / 30} + hours {9} + minutes {30}; + EXPECT_EQ(point, expected); +} + +TEST(TimeTest, FormatsTimestampAsTimeOfDay) { + constexpr std::uint64_t nanos = 34200ull * 1'000'000'000ull; + EXPECT_EQ(itch::format_timestamp(nanos), "09:30:00.000000000"); +} + +TEST(TimeTest, FormatsFullTimePoint) { + using namespace std::chrono; + constexpr std::uint64_t nanos = 34200ull * 1'000'000'000ull; + const auto text = itch::format_time_point(2020y / January / 30, nanos); + EXPECT_NE(text.find("2020-01-30"), std::string::npos); + EXPECT_NE(text.find("09:30:00"), std::string::npos); +} diff --git a/tests/transport/frame_builders.hpp b/tests/transport/frame_builders.hpp new file mode 100644 index 0000000..af5d218 --- /dev/null +++ b/tests/transport/frame_builders.hpp @@ -0,0 +1,246 @@ +#pragma once + +/// @file +/// @brief Test-only fixture-building helpers that construct synthetic +/// ITCH/MoldUDP64/SoupBinTCP/pcap frames for unit tests. +/// +/// These helpers build raw ITCH message payloads and wrap them in the +/// MoldUDP64, SoupBinTCP, and pcap envelopes entirely from in-memory bytes, +/// so the transport decoders can be exercised in tests without any real +/// capture files. This is a test fixture helper, not part of the public +/// library API. +/// +/// @author Bertin Balouki SIMYELI + +#include +#include +#include +#include +#include + +#include "itch/parser.hpp" + +namespace itch::test { + +/// @brief Packs `value` into 2 bytes in big-endian order and appends them to +/// `out`. +/// @param out The byte buffer to append to. +/// @param value The 16-bit value to pack. +inline auto append_be16(std::vector& out, std::uint16_t value) -> void { + const auto big = itch::utils::from_big_endian(value); + std::array bytes {}; + std::memcpy(bytes.data(), &big, sizeof(big)); + out.insert(out.end(), bytes.begin(), bytes.end()); +} + +/// @brief Packs `value` into 4 bytes in big-endian order and appends them to +/// `out`. +/// @param out The byte buffer to append to. +/// @param value The 32-bit value to pack. +inline auto append_be32(std::vector& out, std::uint32_t value) -> void { + const auto big = itch::utils::from_big_endian(value); + std::array bytes {}; + std::memcpy(bytes.data(), &big, sizeof(big)); + out.insert(out.end(), bytes.begin(), bytes.end()); +} + +/// @brief Packs `value` into 8 bytes in big-endian order and appends them to +/// `out`. +/// @param out The byte buffer to append to. +/// @param value The 64-bit value to pack. +inline auto append_be64(std::vector& out, std::uint64_t value) -> void { + const auto big = itch::utils::from_big_endian(value); + std::array bytes {}; + std::memcpy(bytes.data(), &big, sizeof(big)); + out.insert(out.end(), bytes.begin(), bytes.end()); +} + +/// @brief Packs `value` into 4 bytes in little-endian order (a direct copy of +/// the in-memory representation, assuming a little-endian host) and +/// appends them to `out`. +/// @param out The byte buffer to append to. +/// @param value The 32-bit value to pack. +inline auto append_le32(std::vector& out, std::uint32_t value) -> void { + std::array bytes {}; + std::memcpy(bytes.data(), &value, sizeof(value)); + out.insert(out.end(), bytes.begin(), bytes.end()); +} + +/// @brief Appends the raw contents of `src` to `out`. +/// @param out The byte buffer to append to. +/// @param src The bytes to append. +inline auto append_bytes(std::vector& out, const std::vector& src) -> void { + out.insert(out.end(), src.begin(), src.end()); +} + +/// @brief Builds the raw payload (no length prefix) of a System Event message. +/// @param timestamp The 48-bit event timestamp (nanoseconds since midnight) +/// to encode. +/// @param event_code The single-character system event code. +/// @return The encoded System Event message payload. +inline auto system_event_payload(std::uint64_t timestamp, char event_code) + -> std::vector { + std::vector payload; + payload.push_back(std::byte {'S'}); + append_be16(payload, 1); // stock_locate + append_be16(payload, 0); // tracking_number + // 48-bit timestamp: take the low 6 bytes of a big-endian 64-bit value. + std::vector ts; + append_be64(ts, timestamp); + payload.insert(payload.end(), ts.begin() + 2, ts.end()); + payload.push_back(std::byte {static_cast(event_code)}); + return payload; +} + +/// @brief Builds the raw payload (no length prefix) of an Add Order (`A`) +/// message. +/// @param locate The stock locate code. +/// @param ref The order reference number. +/// @param side The single-character side code (`'B'` or `'S'`). +/// @param shares The number of shares in the order. +/// @param symbol The ticker symbol (right-padded/truncated to 8 characters). +/// @param price The raw (unscaled) limit price. +/// @return The encoded Add Order message payload. +inline auto add_order_payload( + std::uint16_t locate, + std::uint64_t ref, + char side, + std::uint32_t shares, + std::string_view symbol, + std::uint32_t price +) -> std::vector { + std::vector payload; + payload.push_back(std::byte {'A'}); + append_be16(payload, locate); + append_be16(payload, 0); // tracking_number + std::vector ts; + append_be64(ts, 1234567); + payload.insert(payload.end(), ts.begin() + 2, ts.end()); // 48-bit timestamp + append_be64(payload, ref); + payload.push_back(std::byte {static_cast(side)}); + append_be32(payload, shares); + for (std::size_t index = 0; index < 8; ++index) { + const char chr = index < symbol.size() ? symbol[index] : ' '; + payload.push_back(std::byte {static_cast(chr)}); + } + append_be32(payload, price); + return payload; +} + +/// @brief Wraps a raw payload as a length-prefixed ITCH frame (and message +/// block, since both use the same 2-byte big-endian length prefix). +/// @param payload The raw payload to prefix. +/// @return The payload prefixed with its 2-byte big-endian length. +inline auto length_prefixed(const std::vector& payload) -> std::vector { + std::vector frame; + append_be16(frame, static_cast(payload.size())); + frame.insert(frame.end(), payload.begin(), payload.end()); + return frame; +} + +/// @brief Builds a complete MoldUDP64 datagram from a list of raw message +/// payloads. +/// @param session The session identifier (right-padded/truncated to 10 +/// characters). +/// @param sequence The sequence number of the first message block. +/// @param payloads The raw message payloads to include, in order. +/// @return The encoded MoldUDP64 datagram. +inline auto moldudp64_packet( + const std::string& session, + std::uint64_t sequence, + const std::vector>& payloads +) -> std::vector { + std::vector packet; + std::array session_field {}; + for (std::size_t index = 0; index < session_field.size(); ++index) { + session_field[index] = index < session.size() ? session[index] : ' '; + } + for (char chr : session_field) { + packet.push_back(std::byte {static_cast(chr)}); + } + append_be64(packet, sequence); + append_be16(packet, static_cast(payloads.size())); + for (const auto& payload : payloads) { + append_bytes(packet, length_prefixed(payload)); + } + return packet; +} + +/// @brief Builds a SoupBinTCP packet (2-byte length + type byte + payload). +/// @param type The single-character SoupBinTCP packet type. +/// @param payload The packet body following the type byte. +/// @return The encoded SoupBinTCP packet. +inline auto soupbin_packet(char type, const std::vector& payload) + -> std::vector { + std::vector packet; + append_be16(packet, static_cast(payload.size() + 1)); + packet.push_back(std::byte {static_cast(type)}); + packet.insert(packet.end(), payload.begin(), payload.end()); + return packet; +} + +/// @brief Wraps a UDP payload in Ethernet/IPv4/UDP headers (a single frame). +/// @param dst_port The destination UDP port to encode in the UDP header. +/// @param payload The UDP payload to wrap. +/// @return The encoded Ethernet/IPv4/UDP frame. +inline auto ethernet_ipv4_udp_frame(std::uint16_t dst_port, const std::vector& payload) + -> std::vector { + std::vector frame; + // Ethernet header: dst(6), src(6), ethertype(2). + for (int index = 0; index < 12; ++index) { + frame.push_back(std::byte {0}); + } + append_be16(frame, 0x0800); // IPv4 + + const std::uint16_t udp_len = static_cast(8 + payload.size()); + const std::uint16_t ip_len = static_cast(20 + udp_len); + + // IPv4 header (20 bytes, no options). + frame.push_back(std::byte {0x45}); // version 4, IHL 5 + frame.push_back(std::byte {0}); // DSCP/ECN + append_be16(frame, ip_len); // total length + append_be16(frame, 0); // identification + append_be16(frame, 0); // flags + fragment offset + frame.push_back(std::byte {64}); // TTL + frame.push_back(std::byte {17}); // protocol = UDP + append_be16(frame, 0); // header checksum (ignored) + append_be32(frame, 0x7F000001); // src ip + append_be32(frame, 0x7F000001); // dst ip + + // UDP header (8 bytes). + append_be16(frame, 12345); // src port + append_be16(frame, dst_port); // dst port + append_be16(frame, udp_len); // length + append_be16(frame, 0); // checksum (ignored) + + frame.insert(frame.end(), payload.begin(), payload.end()); + return frame; +} + +/// @brief Builds a classic little-endian .pcap file from a list of frames. +/// @param frames The captured link-layer frames to include, in order. +/// @return The encoded classic pcap file contents. +inline auto classic_pcap(const std::vector>& frames +) -> std::vector { + std::vector file; + append_le32(file, 0xA1B2C3D4); // magic + file.push_back(std::byte {2}); // version major (LE) + file.push_back(std::byte {0}); + file.push_back(std::byte {4}); // version minor + file.push_back(std::byte {0}); + append_le32(file, 0); // thiszone + append_le32(file, 0); // sigfigs + append_le32(file, 65535); // snaplen + append_le32(file, 1); // network = Ethernet + + for (const auto& frame : frames) { + append_le32(file, 0); // ts_sec + append_le32(file, 0); // ts_usec + append_le32(file, static_cast(frame.size())); // incl_len + append_le32(file, static_cast(frame.size())); // orig_len + file.insert(file.end(), frame.begin(), frame.end()); + } + return file; +} + +} // namespace itch::test diff --git a/tests/transport/test_transport.cpp b/tests/transport/test_transport.cpp new file mode 100644 index 0000000..d518b28 --- /dev/null +++ b/tests/transport/test_transport.cpp @@ -0,0 +1,183 @@ +#include + +#include +#include + +#include "itch/messages.hpp" +#include "itch/transport/moldudp64.hpp" +#include "itch/transport/pcap.hpp" +#include "itch/transport/sequencing.hpp" +#include "itch/transport/soupbintcp.hpp" +#include "transport/frame_builders.hpp" + +namespace { + +using itch::Message; +using itch::SystemEventMessage; + +// Pulls the event_code out of the System Event messages decoded into a vector. +auto event_codes(const std::vector& messages) -> std::string { + std::string codes; + for (const auto& message : messages) { + if (const auto* event = std::get_if(&message)) { + codes.push_back(event->event_code); + } + } + return codes; +} + +} // namespace + +TEST(MoldUdp64, DecodesAllMessageBlocksInPacket) { + std::vector decoded; + itch::transport::MoldUdp64Decoder decoder {[&](const Message& msg) { decoded.push_back(msg); }}; + + const auto packet = itch::test::moldudp64_packet( + "SESSION01", + 1, + {itch::test::system_event_payload(1000, 'O'), itch::test::system_event_payload(2000, 'S')} + ); + + const auto header = decoder.decode_packet(std::span {packet}); + ASSERT_TRUE(header.has_value()); + EXPECT_EQ(header->session_view(), "SESSION01"); + EXPECT_EQ(header->sequence_number, 1U); + EXPECT_EQ(header->message_count, 2U); + EXPECT_EQ(decoder.messages_decoded(), 2U); + EXPECT_EQ(event_codes(decoded), "OS"); +} + +TEST(MoldUdp64, HeartbeatCarriesNoMessages) { + itch::transport::MoldUdp64Decoder decoder {[](const Message&) {}}; + const auto packet = itch::test::moldudp64_packet("SESSION01", 1, {}); + const auto header = decoder.decode_packet(std::span {packet}); + ASSERT_TRUE(header.has_value()); + EXPECT_TRUE(header->is_heartbeat()); + EXPECT_EQ(decoder.messages_decoded(), 0U); +} + +TEST(MoldUdp64, DetectsSequenceGapAcrossPackets) { + itch::transport::MoldUdp64Decoder decoder {[](const Message&) {}}; + std::uint64_t reported_expected = 0; + std::uint64_t reported_received = 0; + decoder.tracker().set_gap_callback( + [&](std::string_view, std::uint64_t expected, std::uint64_t received) { + reported_expected = expected; + reported_received = received; + } + ); + + // Sequences 1 and 2, then jump to 5 (missing 3 and 4). + const auto first = itch::test::moldudp64_packet( + "S", 1, {itch::test::system_event_payload(1, 'O'), itch::test::system_event_payload(2, 'S')} + ); + const auto second = + itch::test::moldudp64_packet("S", 5, {itch::test::system_event_payload(3, 'Q')}); + decoder.decode_packet(std::span {first}); + decoder.decode_packet(std::span {second}); + + EXPECT_EQ(decoder.tracker().gap_count(), 2U); + EXPECT_EQ(reported_expected, 3U); + EXPECT_EQ(reported_received, 5U); +} + +TEST(MoldUdp64, ShortDatagramReturnsNullopt) { + itch::transport::MoldUdp64Decoder decoder {[](const Message&) {}}; + std::vector tiny(4, std::byte {0}); + EXPECT_FALSE(decoder.decode_packet(std::span {tiny}).has_value()); +} + +TEST(SoupBinTcp, DecodesSequencedDataAcrossSegmentBoundaries) { + std::vector decoded; + itch::transport::SoupBinDecoder decoder {[&](const Message& msg) { decoded.push_back(msg); }}; + + const auto packet1 = itch::test::soupbin_packet('S', itch::test::system_event_payload(10, 'O')); + const auto packet2 = itch::test::soupbin_packet('S', itch::test::system_event_payload(20, 'Q')); + + // Concatenate and feed in two chunks that split a packet in half. + std::vector stream; + stream.insert(stream.end(), packet1.begin(), packet1.end()); + stream.insert(stream.end(), packet2.begin(), packet2.end()); + const std::size_t split = packet1.size() + 2; + decoder.feed(std::span {stream.data(), split}); + EXPECT_EQ(decoded.size(), 1U); // Second packet is still incomplete. + decoder.feed(std::span {stream.data() + split, stream.size() - split}); + + EXPECT_EQ(decoder.messages_decoded(), 2U); + EXPECT_EQ(event_codes(decoded), "OQ"); +} + +TEST(SoupBinTcp, LoginAcceptedSetsSessionAndSequence) { + itch::transport::SoupBinDecoder decoder {[](const Message&) {}}; + + std::vector login; + const std::string session = "GLIMPSE "; // 10 chars + for (char chr : session) { + login.push_back(std::byte {static_cast(chr)}); + } + const std::string sequence = " 42"; // 20 chars, right-justified + for (char chr : sequence) { + login.push_back(std::byte {static_cast(chr)}); + } + const auto packet = itch::test::soupbin_packet('A', login); + decoder.feed(std::span {packet}); + + EXPECT_EQ(decoder.current_session(), "GLIMPSE"); + EXPECT_EQ(decoder.next_sequence(), 42U); +} + +TEST(SoupBinTcp, ControlPacketsReachEventCallback) { + itch::transport::SoupBinDecoder decoder {[](const Message&) {}}; + char seen = '\0'; + decoder.set_event_callback([&](itch::transport::SoupBinPacketType type, + std::span) { seen = static_cast(type); }); + const auto heartbeat = itch::test::soupbin_packet('H', {}); + decoder.feed(std::span {heartbeat}); + EXPECT_EQ(seen, 'H'); +} + +TEST(Pcap, ReplaysMessagesFromClassicCapture) { + std::vector decoded; + itch::transport::PcapReader reader {[&](const Message& msg) { decoded.push_back(msg); }}; + + const auto mold = itch::test::moldudp64_packet( + "SESSION01", + 1, + {itch::test::system_event_payload(1, 'O'), itch::test::system_event_payload(2, 'C')} + ); + const auto frame = itch::test::ethernet_ipv4_udp_frame(26477, mold); + const auto file = itch::test::classic_pcap({frame}); + + EXPECT_TRUE(reader.read(std::span {file})); + EXPECT_EQ(reader.udp_datagrams(), 1U); + EXPECT_EQ(reader.messages_decoded(), 2U); + EXPECT_EQ(event_codes(decoded), "OC"); +} + +TEST(Pcap, PortFilterDropsNonMatchingDatagrams) { + std::vector decoded; + itch::transport::PcapReader reader {[&](const Message& msg) { decoded.push_back(msg); }}; + reader.set_udp_port_filter(9999); + + const auto mold = + itch::test::moldudp64_packet("S", 1, {itch::test::system_event_payload(1, 'O')}); + const auto frame = itch::test::ethernet_ipv4_udp_frame(26477, mold); + const auto file = itch::test::classic_pcap({frame}); + + EXPECT_TRUE(reader.read(std::span {file})); + EXPECT_EQ(reader.messages_decoded(), 0U); +} + +TEST(Pcap, UnrecognizedBufferIsRejected) { + itch::transport::PcapReader reader {[](const Message&) {}}; + std::vector junk(64, std::byte {0x11}); + EXPECT_FALSE(reader.read(std::span {junk})); +} + +TEST(SequenceTracker, IgnoresDuplicateReplayedMessages) { + itch::transport::SequenceTracker tracker; + EXPECT_EQ(tracker.observe("S", 1, 3), 0U); // messages 1,2,3 + EXPECT_EQ(tracker.observe("S", 2, 3), 0U); // overlapping replay 2,3,4 + EXPECT_EQ(tracker.messages_seen(), 4U); // only 1,2,3,4 counted once + EXPECT_EQ(tracker.expected_next("S"), 5U); +} diff --git a/tools/itch_tool/CMakeLists.txt b/tools/itch_tool/CMakeLists.txt new file mode 100644 index 0000000..a3d9a85 --- /dev/null +++ b/tools/itch_tool/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.25) + +include(GNUInstallDirs) + +add_executable(itch_tool main.cpp) +set_target_properties(itch_tool PROPERTIES OUTPUT_NAME "itch-tool") + +target_link_libraries( + itch_tool PRIVATE + itch::itch + warnings::strict +) + +install(TARGETS itch_tool RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/tools/itch_tool/main.cpp b/tools/itch_tool/main.cpp new file mode 100644 index 0000000..ca4a6b3 --- /dev/null +++ b/tools/itch_tool/main.cpp @@ -0,0 +1,181 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "itch/io/csv_sink.hpp" +#include "itch/parser.hpp" +#include "itch/transport/pcap.hpp" + +namespace { + +// Writes a formatted line to a stream. Uses std::format (C++20) rather than +// std::println (C++23) so the tool builds on the project's full standard range. +template +auto print_line(std::ostream& out, std::string_view fmt, const Args&... args) -> void { + out << std::vformat(fmt, std::make_format_args(args...)) << '\n'; +} + +// Reads an entire file into a byte buffer. +auto read_file(const std::string& path) -> std::vector { + std::ifstream file {path, std::ios::binary}; + if (!file) { + return {}; + } + std::vector raw {std::istreambuf_iterator {file}, {}}; + std::vector bytes(raw.size()); + if (!raw.empty()) { + std::memcpy(bytes.data(), raw.data(), raw.size()); + } + return bytes; +} + +// Drives a callback over every message in a file, auto-detecting whether the +// input is a pcap/pcapng capture or a raw length-prefixed ITCH stream. +auto for_each_message(std::span data, const itch::MessageCallback& callback) + -> void { + itch::transport::PcapReader reader {callback}; + if (reader.read(data)) { + return; // The input was a capture file. + } + itch::Parser parser; + parser.parse(data, callback); +} + +// Parses a comma-or-concatenated list of message-type characters (e.g. "AEP"). +auto parse_type_filter(std::string_view spec) -> std::array { + std::array wanted {}; + for (char chr : spec) { + if (chr != ',') { + wanted[static_cast(chr)] = true; + } + } + return wanted; +} + +auto message_type_of(const itch::Message& message) -> char { + return std::visit([](const auto& msg) { return msg.message_type; }, message); +} + +auto cmd_stats(std::span data) -> int { + std::array counts {}; + std::uint64_t total = 0; + for_each_message(data, [&](const itch::Message& msg) { + ++counts[static_cast(message_type_of(msg))]; + ++total; + }); + print_line(std::cout, "{:<6} {:>14}", "type", "count"); + for (std::size_t type = 0; type < counts.size(); ++type) { + if (counts[type] > 0) { + print_line(std::cout, "{:<6} {:>14}", static_cast(type), counts[type]); + } + } + print_line(std::cout, "{:<6} {:>14}", "TOTAL", total); + return 0; +} + +auto cmd_inspect(std::span data, std::uint64_t limit) -> int { + std::uint64_t shown = 0; + for_each_message(data, [&](const itch::Message& msg) { + if (shown < limit) { + std::cout << msg << '\n'; // Message provides operator<<. + ++shown; + } + }); + print_line(std::cout, "(showed {} messages)", shown); + return 0; +} + +auto cmd_filter_or_convert( + std::span data, + const std::array& wanted, + bool has_filter, + const std::string& out_path +) -> int { + std::ofstream file_out; + if (!out_path.empty()) { + file_out.open(out_path, std::ios::binary); + if (!file_out) { + print_line(std::cerr, "Error: cannot open output '{}'.", out_path); + return 1; + } + } + std::ostream& out = out_path.empty() ? std::cout : file_out; + itch::io::CsvSink sink {out}; + for_each_message(data, [&](const itch::Message& msg) { + if (!has_filter || wanted[static_cast(message_type_of(msg))]) { + sink.write(msg); + } + }); + sink.flush(); + print_line(std::cerr, "Wrote {} rows.", sink.rows_written()); + return 0; +} + +auto usage(const char* program) -> int { + print_line(std::cerr, "itch-tool - inspect, filter, and convert NASDAQ ITCH 5.0 data"); + print_line(std::cerr, "Usage:"); + print_line(std::cerr, " {} stats ", program); + print_line(std::cerr, " {} inspect [--limit N]", program); + print_line(std::cerr, " {} filter --types [--out ]", program); + print_line(std::cerr, " {} convert [--to csv] [--out ]", program); + print_line(std::cerr, "Input may be a raw ITCH stream or a .pcap/.pcapng capture."); + return 1; +} + +} // namespace + +auto main(int argc, char* argv[]) -> int { + const std::vector args {argv, argv + argc}; + if (args.size() < 3) { + return usage(argv[0]); + } + const std::string& command = args[1]; + const std::string& path = args[2]; + + std::string out_path; + std::string types; + std::uint64_t limit = 20; + std::array wanted {}; + bool has_filter = false; + for (std::size_t index = 3; index < args.size(); ++index) { + if (args[index] == "--out" && index + 1 < args.size()) { + out_path = args[++index]; + } else if (args[index] == "--types" && index + 1 < args.size()) { + types = args[++index]; + wanted = parse_type_filter(types); + has_filter = true; + } else if (args[index] == "--limit" && index + 1 < args.size()) { + limit = std::stoull(args[++index]); + } else if (args[index] == "--to" && index + 1 < args.size()) { + ++index; // Only csv is supported without the Arrow build option. + } + } + + const std::vector data = read_file(path); + if (data.empty()) { + print_line(std::cerr, "Error: cannot read '{}' (missing or empty).", path); + return 1; + } + const std::span view {data}; + + if (command == "stats") { + return cmd_stats(view); + } + if (command == "inspect") { + return cmd_inspect(view, limit); + } + if (command == "filter") { + return cmd_filter_or_convert(view, wanted, has_filter, out_path); + } + if (command == "convert") { + return cmd_filter_or_convert(view, wanted, has_filter, out_path); + } + return usage(argv[0]); +} diff --git a/vcpkg.json b/vcpkg.json index 165ef1b..f268b88 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -3,5 +3,24 @@ "gtest", "benchmark" ], + "features": { + "python": { + "description": "Build the pybind11 Python bindings", + "dependencies": [ + "pybind11" + ] + }, + "arrow": { + "description": "Enable Apache Arrow / Parquet columnar export", + "dependencies": [ + { + "name": "arrow", + "features": [ + "parquet" + ] + } + ] + } + }, "builtin-baseline": "5bf0c55239da398b8c6f450818c9e28d36bf9966" }