diff --git a/CMakeLists.txt b/CMakeLists.txt index 1f9636a..4285f73 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,6 +12,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON) option(IRONPHOENIX_BUILD_TESTS "Build IronPhoenix correctness tests" ON) option(IRONPHOENIX_BUILD_BENCHMARK "Build IronPhoenix microbenchmark" ON) +option(IRONPHOENIX_BUILD_DATASET_GENERATOR "Build PhoenixNet dataset generator" ON) add_library(ironphoenix_core STATIC src/geometry.cpp @@ -22,13 +23,16 @@ add_library(ironphoenix_core STATIC src/lmr.cpp src/tt.cpp src/eval.cpp + src/nnue.cpp src/search.cpp + src/multipv.cpp src/fen.cpp src/movegen.cpp src/preft.cpp src/uci.cpp include/ironphoenix/eval.hpp include/ironphoenix/lmr.hpp + include/ironphoenix/nnue.hpp ) add_library(IronPhoenix::Core ALIAS ironphoenix_core) @@ -115,6 +119,46 @@ if(IRONPHOENIX_IPO_SUPPORTED) set_property(TARGET ironphoenix PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE) endif() +if(IRONPHOENIX_BUILD_DATASET_GENERATOR) + add_executable(ironphoenix_dataset + tools/nnue_dataset_resumable.cpp + ) + target_link_libraries(ironphoenix_dataset PRIVATE IronPhoenix::Core) + target_compile_features(ironphoenix_dataset PRIVATE cxx_std_20) + set_target_properties(ironphoenix_dataset PROPERTIES + OUTPUT_NAME "ironphoenix_dataset" + FOLDER "Tools" + ) + + if(MSVC) + target_compile_options(ironphoenix_dataset PRIVATE + /W4 + /permissive- + /Zc:__cplusplus + /MP + $<$:/O2> + $<$:/Oi> + $<$:/Ot> + $<$:/O2> + $<$:/Oi> + $<$:/Ot> + ) + else() + target_compile_options(ironphoenix_dataset PRIVATE + -Wall + -Wextra + -Wpedantic + $<$:-O3> + $<$:-O3> + ) + endif() + + if(IRONPHOENIX_IPO_SUPPORTED) + set_property(TARGET ironphoenix_dataset PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) + set_property(TARGET ironphoenix_dataset PROPERTY INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO TRUE) + endif() +endif() + if(IRONPHOENIX_BUILD_TESTS) include(CTest) enable_testing() @@ -139,4 +183,4 @@ if(IRONPHOENIX_BUILD_BENCHMARK) target_link_libraries(ironphoenix_bench PRIVATE IronPhoenix::Core) target_compile_features(ironphoenix_bench PRIVATE cxx_std_20) set_target_properties(ironphoenix_bench PROPERTIES FOLDER "Benchmarks") -endif() \ No newline at end of file +endif() diff --git a/README.md b/README.md index 7378027..1a29e5a 100644 --- a/README.md +++ b/README.md @@ -1 +1,273 @@ -# IronPhoenix \ No newline at end of file +# ๐Ÿ”ฅ IronPhoenix + +**IronPhoenix** is a high-performance **4-player chess Teams engine** written in modern C++20. + +It is built around the chess.com-style 4PC cross board, team-aware search, fast move generation, handcrafted evaluation, and **PhoenixNet**, a purpose-built NNUE evaluator for four-player Teams chess. + +> โค๏ธ Built with love by Nick. + +--- + +## โ™Ÿ๏ธ What IronPhoenix Plays + +IronPhoenix is designed for **4-player Teams chess**: + +- **Red + Yellow** vs **Blue + Green** +- Turn order: **Red โ†’ Blue โ†’ Yellow โ†’ Green** +- 14ร—14 cross-shaped board +- 160 playable squares +- Team-aware king safety and terminal king-capture handling +- Four-player castling, promotion, en-passant, and move-generation rules +- Four-player FEN parsing and position setup + +The engine evaluates positions from the **side-to-move team perspective**: positive scores favor the team whose player is currently moving. + +--- + +## โšก Engine Features + +### ๐Ÿ”Ž Search + +IronPhoenix includes a modern alpha-beta search stack designed specifically for four-player Teams chess: + +- Negamax / PVS-style search +- Quiescence search +- Transposition tables +- History-based move ordering +- Static Exchange Evaluation (SEE) +- Late Move Reductions (LMR) +- Search pruning and reductions +- Iterative deepening +- Principal variation reporting +- Time / depth search limits +- MultiPV root search + +`MultiPV=1` keeps the normal single-PV search path. Higher values use the dedicated MultiPV root search. + +```text +setoption name MultiPV value 4 +go depth 8 +``` + +--- + +## ๐Ÿงฎ Evaluation + +IronPhoenix supports two evaluation paths. + +### ๐Ÿ› ๏ธ Handcrafted Evaluation + +The HCE currently provides the engine's deterministic fallback and PhoenixNet teacher foundation, including: + +- Material +- Mobility +- Team-relative scoring + +### ๐Ÿง  PhoenixNet NNUE + +**PhoenixNet** is IronPhoenix's native neural evaluator. + +Current PhoenixNet v1 architecture: + +- 65,280 sparse input features +- 17 king buckets +- 4 relative piece colors +- 6 piece types +- Own-king feature stream +- Partner-king feature stream +- 128-wide shared feature transformer +- `256 โ†’ 32 โ†’ 32 โ†’ 1` dense network +- Centipawn-like side-to-move-team output + +If `ironphoenix.nnue` is available beside the executable, IronPhoenix loads it automatically. Otherwise it falls back to HCE. + +For the full PhoenixNet architecture, dataset format, training flow, export format, append/resume support, and generator options, see **[๐Ÿง  PhoenixNet NNUE documentation](nnue/README.md)**. + +--- + +## ๐Ÿ—๏ธ Building IronPhoenix + +### ๐Ÿ“‹ Requirements + +- CMake **3.24+** +- A C++20 compiler +- Visual Studio / MSVC, Clang, or GCC +- Threads support + +### ๐ŸชŸ Windows / Visual Studio + +From the project root: + +```powershell +cmake -S . -B build -DIRONPHOENIX_BUILD_DATASET_GENERATOR=ON +cmake --build build --config Release +``` + +To build only the engine: + +```powershell +cmake --build build --config Release --target ironphoenix +``` + +To build only the PhoenixNet dataset generator: + +```powershell +cmake --build build --config Release --target ironphoenix_dataset +``` + +If you are using the Visual Studio CMake `x64-release` layout used during development: + +```cmd +cmake --build out\build\x64-release --target ironphoenix +cmake --build out\build\x64-release --target ironphoenix_dataset +``` + +--- + +## โ–ถ๏ธ Running the Engine + +Launch the engine executable and enter UCI-style commands: + +```text +uci +isready +position startpos +go depth 8 +``` + +Enable MultiPV: + +```text +setoption name MultiPV value 4 +position startpos +go depth 8 +``` + +Useful engine options include: + +```text +setoption name Hash value 256 +setoption name MultiPV value 4 +setoption name Clear Hash +``` + +--- + +## ๐Ÿงช PhoenixNet Dataset Generation + +The project builds a native dataset generator: + +```text +ironphoenix_dataset +``` + +It generates self-play positions using the engine search as a teacher and stores the exact sparse features consumed by PhoenixNet. + +### ๐Ÿš€ Example dataset run + +```cmd +ironphoenix_dataset.exe --positions 500000 --depth 8 --workers 8 --hash 16 --random-plies 8 --skip-plies 12 --sample-every 3 --exploration 0.15 --exploration-multipv 4 --exploration-temperature 120 --exploration-max-loss 250 --dedup --output "..\..\..\nnue\phoenix-v1.ipd" +``` + +The generator supports: + +- Parallel self-play workers +- Controlled MultiPV exploration +- Single-PV search for non-exploration positions +- NNUE-feature deduplication +- Randomized opening plies +- Native compact `.ipd` datasets +- Safe append mode +- Resume-to-total mode +- Graceful `Ctrl+C` stopping +- Recovery of complete records from older interrupted runs +- Progress reporting every 10 saved positions + +### ๐Ÿ›‘ Stop and resume safely + +Press **Ctrl+C once**. The generator stops active searches, joins its workers, flushes records, and commits the current dataset count. + +Resume toward a total target: + +```cmd +ironphoenix_dataset.exe --resume --positions 500000 --depth 8 --workers 8 --hash 16 --dedup --output "..\..\..\nnue\phoenix-v1.ipd" +``` + +If the file already contains 50,000 positions, this produces only the remaining 450,000. + +--- + +## ๐Ÿ‹๏ธ Training PhoenixNet + +Create a Python environment: + +```powershell +py -m venv nnue\.venv +.\nnue\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +pip install torch numpy +``` + +Train: + +```powershell +python nnue\train.py nnue\phoenix-v1.ipd ` + --epochs 20 ` + --batch-size 1024 ` + --lr 0.001 ` + -o nnue\phoenixnet-v1.pt +``` + +Export to the native IronPhoenix NNUE format: + +```powershell +python nnue\export.py nnue\phoenixnet-v1.pt ` + -o ironphoenix.nnue ` + --output-scale 400 +``` + +Place `ironphoenix.nnue` beside `ironphoenix.exe` and start the engine. A successful load prints: + +```text +PhoenixNet loaded: ironphoenix.nnue +``` + +--- + +## ๐Ÿ“ Project Layout + +```text +IronPhoenix/ +โ”œโ”€โ”€ include/ironphoenix/ # Public engine headers +โ”œโ”€โ”€ src/ # Engine implementation +โ”œโ”€โ”€ tools/ # Dataset generation tools +โ”œโ”€โ”€ tests/ # Correctness tests and benchmarks +โ”œโ”€โ”€ nnue/ # PhoenixNet model, trainer, exporter, docs +โ”œโ”€โ”€ CMakeLists.txt # Build configuration +โ””โ”€โ”€ README.md # Project overview +``` + +--- + +## ๐Ÿงญ Development Direction + +IronPhoenix is under active development. Current and planned work includes: + +- Stronger handcrafted evaluation terms +- Larger and more diverse PhoenixNet datasets +- Incremental NNUE accumulators +- Quantized NNUE weights +- SIMD / AVX2 inference +- Search tuning and SPRT testing +- Opening-book / explorer integration +- Continued search-performance work + +PhoenixNet v1 intentionally keeps a straightforward full-refresh inference path as a correctness baseline before aggressive optimization. + +--- + +## โค๏ธ Author + +**IronPhoenix** is developed by **Nick / KnightStackSys** as a dedicated high-performance engine for 4-player Teams chess. + +๐Ÿ”ฅ **Build. Search. Learn. Rise.** diff --git a/include/ironphoenix/nnue.hpp b/include/ironphoenix/nnue.hpp new file mode 100644 index 0000000..a63197b --- /dev/null +++ b/include/ironphoenix/nnue.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include "types.hpp" + +#include +#include +#include + +namespace ironphoenix { + +class Position; + +namespace NNUE { + +constexpr std::size_t KING_BUCKETS = 17; +constexpr std::size_t RELATIVE_COLORS = 4; +constexpr std::size_t NNUE_PIECE_TYPES = 6; +constexpr std::size_t FEATURE_COUNT = KING_BUCKETS * RELATIVE_COLORS * NNUE_PIECE_TYPES * SQUARE_NB; +constexpr std::size_t FT_SIZE = 128; +constexpr std::size_t DENSE_INPUT = FT_SIZE * 2; +constexpr std::size_t HIDDEN1_SIZE = 32; +constexpr std::size_t HIDDEN2_SIZE = 32; + +// Reserved for the upcoming quantized/incremental implementation. +constexpr int ACTIVATION_MAX = 127; +constexpr int HIDDEN_SHIFT = 6; +constexpr int OUTPUT_DIVISOR = 64; + +// PhoenixNet evaluates from the current side-to-move player's perspective. +// Positive values mean an advantage for side-to-move's team. +[[nodiscard]] bool loadNetwork(const std::string& path) noexcept; +void unloadNetwork() noexcept; +[[nodiscard]] bool loaded() noexcept; +[[nodiscard]] const std::string& loadedPath() noexcept; + +[[nodiscard]] int evaluate(const Position& pos) noexcept; + +// Public helpers are useful for dataset generation and correctness tests. +[[nodiscard]] Square canonicalSquare(Color perspective, Square sq) noexcept; +[[nodiscard]] unsigned relativeColor(Color perspective, Color pieceColor) noexcept; +[[nodiscard]] unsigned kingBucket(Color perspective, Square kingSq) noexcept; +[[nodiscard]] std::size_t featureIndex( + Color perspective, + Square anchorKing, + Piece piece, + Square pieceSq) noexcept; + +} // namespace NNUE +} // namespace ironphoenix diff --git a/include/ironphoenix/search.hpp b/include/ironphoenix/search.hpp index 82c9c9c..0604b00 100644 --- a/include/ironphoenix/search.hpp +++ b/include/ironphoenix/search.hpp @@ -35,6 +35,8 @@ namespace ironphoenix { class SearchEngine { public: + static constexpr int MAX_MULTI_PV = 32; + SearchEngine(); ~SearchEngine(); @@ -42,6 +44,11 @@ namespace ironphoenix { SearchEngine& operator=(const SearchEngine&) = delete; void start(Position position, SearchLimits limits, std::ostream& out); + + // Dedicated root MultiPV search. MultiPV=1 should continue to use + // start(), preserving the normal PVS/aspiration search path exactly. + void startMultiPV(Position position, SearchLimits limits, int multiPV, std::ostream& out); + void stop() noexcept; void stopAndWait() noexcept; @@ -76,6 +83,8 @@ namespace ironphoenix { mutable std::mutex outputMutex_; void run(Position position, std::ostream& out); + void runMultiPV(Position position, int multiPV, std::ostream& out); + int negamax(Position& pos, int depth, int alpha, int beta, int ply, bool pvNode); int qsearch(Position& pos, int alpha, int beta, int ply, bool pvNode); int evaluate(const Position& pos) const noexcept; @@ -102,6 +111,16 @@ namespace ironphoenix { int pvLength, ScoreBound bound = ScoreBound::Exact ); + + void emitMultiPVInfo( + std::ostream& out, + int depth, + int multiPvIndex, + int score, + const Move* pv, + int pvLength + ); + void emitBestMove(std::ostream& out, Move move); }; diff --git a/include/ironphoenix/uci.hpp b/include/ironphoenix/uci.hpp index 18be569..a1ca94b 100644 --- a/include/ironphoenix/uci.hpp +++ b/include/ironphoenix/uci.hpp @@ -29,6 +29,7 @@ namespace ironphoenix { Fen4State fenState_{}; std::string startFen_; SetupType setup_ = SetupType::Modern; + int multiPV_ = 1; SearchEngine search_{}; bool handleLine(const std::string& line, std::ostream& out); diff --git a/ironphoenix.nnue b/ironphoenix.nnue new file mode 100644 index 0000000..a2ea263 Binary files /dev/null and b/ironphoenix.nnue differ diff --git a/nnue/.gitignore b/nnue/.gitignore new file mode 100644 index 0000000..6f596e1 --- /dev/null +++ b/nnue/.gitignore @@ -0,0 +1,220 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[codz] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py.cover +*.lcov +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +# Pipfile.lock + +# uv +# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# uv.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +# poetry.lock +# poetry.toml + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. +# https://pdm-project.org/en/latest/usage/project/#working-with-version-control +# pdm.lock +# pdm.toml +.pdm-python +.pdm-build/ + +# pixi +# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. +# pixi.lock +# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one +# in the .venv directory. It is recommended not to include this directory in version control. +.pixi/* +!.pixi/config.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule* +celerybeat.pid + +# Redis +*.rdb +*.aof +*.pid + +# RabbitMQ +mnesia/ +rabbitmq/ +rabbitmq-data/ + +# ActiveMQ +activemq-data/ + +# SageMath parsed files +*.sage.py + +# Environments +.env +.envrc +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs/Zensical documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +# .idea/ + +# Abstra +# Abstra is an AI-powered process automation framework. +# Ignore directories containing user credentials, local state, and settings. +# Learn more at https://abstra.io/docs +.abstra/ + +# Visual Studio Code +# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore that +# can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore +# and can be added to the global gitignore or merged into this file. However, if you prefer, you +# could uncomment the following to ignore the entire vscode folder +# .vscode/ +# Temporary file for partial code execution +tempCodeRunnerFile.py + +# Ruff stuff: +.ruff_cache/ + +# PyPI configuration file +.pypirc + +# Marimo +marimo/_static/ +marimo/_lsp/ +__marimo__/ + +# Streamlit +.streamlit/secrets.toml diff --git a/nnue/README.md b/nnue/README.md index 8b13789..682cd06 100644 --- a/nnue/README.md +++ b/nnue/README.md @@ -1 +1,280 @@ +# ๐Ÿง  PhoenixNet NNUE v1 +PhoenixNet is IronPhoenix's first neural evaluator for four-player Teams chess. + +## ๐Ÿ—๏ธ Architecture + +- 160 playable squares on the 14x14 cross board +- 4 relative colors: self, next enemy, partner, previous enemy +- 6 piece types +- 17 king buckets: 16 spatial buckets plus one missing/eliminated-king bucket +- shared sparse feature transformer: `65280 -> 128` +- two accumulator streams evaluated from side-to-move's perspective: + - current player's king + - partner king +- concatenated dense input: `256` +- hidden layers: `32 -> 32` +- scalar output converted to a centipawn-like score + +The engine score is always side-to-move-team relative. Positive values favor the team of the player whose turn it is. + +## ๐Ÿงฉ Feature Index + +Each sparse feature is: + +```text +king bucket ร— relative piece color ร— piece type ร— canonical piece square +``` + +Positions are rotated into the current player's orientation before indexing. Red is canonical, Blue rotates 90 degrees, Yellow 180 degrees, and Green 270 degrees. + +## โš™๏ธ Runtime Behavior + +IronPhoenix tries to load `ironphoenix.nnue` from its working directory at startup. + +You can override the path with: + +```text +IRONPHOENIX_NNUE=/path/to/network.nnue +``` + +If no valid network is found, the engine automatically falls back to the existing handcrafted material + mobility evaluator. + +# ๐Ÿ”€ MultiPV + +IronPhoenix exposes a UCI `MultiPV` option: + +```text +setoption name MultiPV value 4 +``` + +Valid values are `1..32`. `MultiPV=1` keeps the original PVS/aspiration search path unchanged. Values greater than one use a dedicated root MultiPV search and emit normal UCI lines such as: + +```text +info depth 8 seldepth 18 multipv 1 score cp 74 ... pv ... +info depth 8 seldepth 18 multipv 2 score cp 51 ... pv ... +info depth 8 seldepth 18 multipv 3 score cp 29 ... pv ... +info depth 8 seldepth 18 multipv 4 score cp 12 ... pv ... +``` + +The first line is the best move and is still returned as `bestmove`. + +# ๐Ÿงช Dataset Generation + +The CMake build produces `ironphoenix_dataset` when `IRONPHOENIX_BUILD_DATASET_GENERATOR=ON`. + +The generator uses HCE-backed search as the teacher, writes exact C++ `NNUE::featureIndex()` sparse features, supports deduplication, controlled exploration, append/resume, and graceful Ctrl+C stopping. + +### ๐Ÿงซ Quick smoke dataset + +```powershell +.\build\Release\ironphoenix_dataset.exe ` + --positions 10000 ` + --depth 6 ` + --workers 4 ` + --hash 16 ` + --output nnue\phoenix-smoke.ipd +``` + +### ๐Ÿš€ Larger first training set + +```powershell +.\build\Release\ironphoenix_dataset.exe ` + --positions 500000 ` + --depth 8 ` + --workers 8 ` + --hash 16 ` + --random-plies 8 ` + --skip-plies 12 ` + --sample-every 3 ` + --exploration 0.15 ` + --exploration-multipv 4 ` + --exploration-temperature 120 ` + --exploration-max-loss 250 ` + --output nnue\phoenix-v1.ipd +``` + +On a 16-core / 24-thread CPU, start around 8-12 generator workers. Each worker owns its own search engine and transposition table. `--hash` is per worker, so 12 workers with `--hash 16` use roughly 192 MB just for transposition tables. + +## โšก Faster Exploration Scheduling + +Exploration no longer forces every teacher search through MultiPV. + +With `--exploration 0.15`: + +- about 85% of eligible teacher searches stay normal single-PV searches +- about 15% are selected up front for MultiPV exploration +- exploration is limited to ply 96 and earlier +- positions after ply 96 always use single-PV search +- if an exploration search has no acceptable alternative within `--exploration-max-loss`, the best move is still used +- the training target always remains MultiPV #1's evaluation + +This preserves diversity while avoiding the old behavior where MultiPV was paid for on 100% of searches even though only a small fraction actually explored. + +Set `--exploration 0` for pure single-PV generation. + +## ๐ŸŽฒ Opening Diversity and Deduplication + +The starting position is never sampled. The default opening phase uses 8 random non-terminal plies and sampling starts no earlier than ply 12. + +Deduplication is enabled by default. The generator fingerprints the exact PhoenixNet own-king and partner-king feature streams plus side to move. Duplicate neural inputs are skipped and generation continues until the requested number of unique positions is reached. + +Append/resume preloads fingerprints from the existing dataset so deduplication works across runs. + +## ๐Ÿ›‘ Graceful Stop and Resume + +Press `Ctrl+C` once to stop safely. Active searches are stopped, worker threads exit, record bytes are flushed, and the authoritative record count is committed before the process returns. + +Resume toward a total target: + +```powershell +.\build\Release\ironphoenix_dataset.exe ` + --resume ` + --positions 500000 ` + --depth 8 ` + --workers 8 ` + --hash 16 ` + --exploration 0.15 ` + --exploration-multipv 4 ` + --output nnue\phoenix-v1.ipd +``` + +If 50,000 positions already exist, `--resume --positions 500000` generates the remaining 450,000. + +`--append --positions 500000` has different semantics: it adds another 500,000 positions. + +๐Ÿ“Š Progress is printed every 10 saved positions. + +Older abruptly interrupted files with stale header counts are scanned for complete trailing records. Complete records are recovered; only an incomplete final record is discarded. + +## ๐Ÿ› ๏ธ Useful Generator Options + +```text +--positions N number of samples (or total target with --resume) +--append add N new samples to an existing compatible .ipd +--resume continue toward N total samples +--depth N teacher search depth +--workers N parallel self-play workers +--hash MB transposition-table size per worker +--random-plies N random legal opening plies +--skip-plies N earliest configured sample ply +--sample-every N save one of every N searched positions +--max-plies N maximum game length +--cp-clamp N teacher CP clamp +--score-scale N CP represented by one network output unit +--exploration P probability of a MultiPV exploration search +--exploration-multipv N number of root lines available to exploration +--exploration-temperature N softmax temperature in centipawns +--exploration-max-loss N reject alternatives worse than best by more than N CP +--dedup enable position-feature deduplication (default) +--no-dedup disable deduplication +--seed N deterministic RNG seed +``` + +## ๐Ÿ’พ Native IPD Dataset Format + +`ironphoenix_dataset` writes a compact little-endian `IPDATA1` stream. Each record contains: + +- own-king sparse feature indices +- partner-king sparse feature indices +- scaled training target +- original teacher centipawn score +- game ID +- ply +- side to move +- flags for check, eliminated-player state, and whether the continuation move was exploratory + +Feature indices are stored as `uint16` because PhoenixNet v1 has 65,280 features. + +`train.py` reads `.ipd` directly and uses game IDs to keep entire self-play games on one side of the train/validation split. + +# ๐Ÿ‹๏ธ Training + +Install Python dependencies: + +```powershell +py -m venv nnue\.venv +.\nnue\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +pip install torch numpy +``` + +### ๐Ÿงช Train a smoke network + +```powershell +python nnue\train.py nnue\phoenix-smoke.ipd ` + --epochs 5 ` + --batch-size 1024 ` + -o nnue\phoenix-smoke.pt +``` + +### ๐Ÿ”ฅ Train the larger network + +```powershell +python nnue\train.py nnue\phoenix-v1.ipd ` + --epochs 20 ` + --batch-size 1024 ` + --lr 0.001 ` + -o nnue\phoenixnet-v1.pt +``` + +If CUDA is available, `train.py` selects it automatically. Use `--device cpu` or `--device cuda` to override. + +# ๐Ÿ“ฆ Exporting the Network + +Export the best checkpoint to the format consumed by the engine: + +```powershell +python nnue\export.py nnue\phoenixnet-v1.pt ` + -o ironphoenix.nnue ` + --output-scale 400 +``` + +Place `ironphoenix.nnue` beside `ironphoenix.exe`, or set `IRONPHOENIX_NNUE` to its full path. + +At startup a successful load prints: + +```text +PhoenixNet loaded: ironphoenix.nnue +``` + +If the file is missing or incompatible, IronPhoenix prints the HCE fallback message and continues using material + mobility. + +## ๐Ÿงฑ NNUE Binary File Format + +All values are little-endian. PhoenixNet v1 currently stores float32 weights to keep training/export validation straightforward. + +```text +char[8] magic = "IPNNUE1\0" +u32 version = 1 +u32 feature_count = 65280 +u32 ft_size = 128 +u32 hidden1_size = 32 +u32 hidden2_size = 32 +f32 output_scale + +f32[65280 * 128] feature_transformer_weights +f32[128] feature_transformer_bias +f32[32 * 256] hidden1_weights +f32[32] hidden1_bias +f32[32 * 32] hidden2_weights +f32[32] hidden2_bias +f32[32] output_weights +f32 output_bias +``` + +## ๐Ÿ—บ๏ธ Development Stages + +PhoenixNet v1 intentionally starts with full accumulator refreshes. This gives a correctness baseline before optimization. + +Next stages: + +1. add make/undo incremental accumulator updates +2. maintain four seat perspectives and own/partner king streams +3. refresh only streams whose king bucket changes +4. quantize feature-transformer weights to int16 +5. quantize dense weights to int8 with int32 accumulation +6. add AVX2 inference +7. SPRT optimized implementations against the reference implementation + +Do not remove the refresh implementation until incremental make/undo tests prove eval equivalence across captures, promotions, en-passant, castling, and king bucket changes. diff --git a/nnue/export.py b/nnue/export.py new file mode 100644 index 0000000..be9e9aa --- /dev/null +++ b/nnue/export.py @@ -0,0 +1,79 @@ +"""Export a trained PhoenixNet checkpoint to IronPhoenix's .nnue format.""" + +from __future__ import annotations + +import argparse +import struct +from pathlib import Path + +import torch + +from model import ( + FEATURE_COUNT, + FT_SIZE, + HIDDEN1_SIZE, + HIDDEN2_SIZE, + PhoenixNet, +) + +MAGIC = b"IPNNUE1\0" +VERSION = 1 + + +def _write_tensor(file, tensor: torch.Tensor) -> None: + data = tensor.detach().cpu().contiguous().float().numpy().tobytes(order="C") + file.write(data) + + +def export_network(model: PhoenixNet, output: Path, output_scale: float = 400.0) -> None: + model.eval() + + with output.open("wb") as f: + f.write(MAGIC) + f.write(struct.pack( + " PhoenixNet: + model = PhoenixNet() + checkpoint = torch.load(path, map_location="cpu") + + if isinstance(checkpoint, dict) and "model" in checkpoint: + state_dict = checkpoint["model"] + else: + state_dict = checkpoint + + model.load_state_dict(state_dict) + return model + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint", type=Path) + parser.add_argument("-o", "--output", type=Path, default=Path("ironphoenix.nnue")) + parser.add_argument("--output-scale", type=float, default=400.0) + args = parser.parse_args() + + model = load_checkpoint(args.checkpoint) + export_network(model, args.output, args.output_scale) + print(f"exported {args.output}") + + +if __name__ == "__main__": + main() diff --git a/nnue/model.py b/nnue/model.py new file mode 100644 index 0000000..dc4064a --- /dev/null +++ b/nnue/model.py @@ -0,0 +1,73 @@ +"""PhoenixNet v1 training model for IronPhoenix 4-player Teams chess.""" + +from __future__ import annotations + +import torch +from torch import Tensor, nn + +KING_BUCKETS = 17 +RELATIVE_COLORS = 4 +PIECE_TYPES = 6 +SQUARES = 160 +FEATURE_COUNT = KING_BUCKETS * RELATIVE_COLORS * PIECE_TYPES * SQUARES +FT_SIZE = 128 +HIDDEN1_SIZE = 32 +HIDDEN2_SIZE = 32 + + +class PhoenixNet(nn.Module): + """Dual-allied-king sparse NNUE network. + + Each position supplies two sparse feature bags from the side-to-move + perspective: one anchored on its own king and one anchored on its partner's + king. The embedding/feature-transformer weights are shared by both streams. + """ + + def __init__(self) -> None: + super().__init__() + + self.feature_transformer = nn.EmbeddingBag( + FEATURE_COUNT, + FT_SIZE, + mode="sum", + include_last_offset=False, + ) + self.ft_bias = nn.Parameter(torch.zeros(FT_SIZE)) + + self.hidden1 = nn.Linear(FT_SIZE * 2, HIDDEN1_SIZE) + self.hidden2 = nn.Linear(HIDDEN1_SIZE, HIDDEN2_SIZE) + self.output = nn.Linear(HIDDEN2_SIZE, 1) + + self.reset_parameters() + + def reset_parameters(self) -> None: + nn.init.normal_(self.feature_transformer.weight, mean=0.0, std=0.01) + nn.init.zeros_(self.ft_bias) + nn.init.xavier_uniform_(self.hidden1.weight) + nn.init.zeros_(self.hidden1.bias) + nn.init.xavier_uniform_(self.hidden2.weight) + nn.init.zeros_(self.hidden2.bias) + nn.init.xavier_uniform_(self.output.weight) + nn.init.zeros_(self.output.bias) + + @staticmethod + def crelu(x: Tensor) -> Tensor: + return torch.clamp(x, 0.0, 1.0) + + def _transform(self, indices: Tensor, offsets: Tensor) -> Tensor: + return self.crelu(self.feature_transformer(indices, offsets) + self.ft_bias) + + def forward( + self, + own_indices: Tensor, + own_offsets: Tensor, + partner_indices: Tensor, + partner_offsets: Tensor, + ) -> Tensor: + own = self._transform(own_indices, own_offsets) + partner = self._transform(partner_indices, partner_offsets) + + x = torch.cat((own, partner), dim=1) + x = self.crelu(self.hidden1(x)) + x = self.crelu(self.hidden2(x)) + return self.output(x).squeeze(1) diff --git a/nnue/phoenix-test.pt b/nnue/phoenix-test.pt new file mode 100644 index 0000000..a2fbed4 Binary files /dev/null and b/nnue/phoenix-test.pt differ diff --git a/nnue/phoenix-v1.ipd b/nnue/phoenix-v1.ipd new file mode 100644 index 0000000..8d75829 Binary files /dev/null and b/nnue/phoenix-v1.ipd differ diff --git a/nnue/train.py b/nnue/train.py new file mode 100644 index 0000000..174a7de --- /dev/null +++ b/nnue/train.py @@ -0,0 +1,327 @@ +"""Train PhoenixNet from IronPhoenix sparse NNUE datasets. + +Supported inputs: + +1. Native IronPhoenix .ipd files produced by ironphoenix_dataset. +2. Legacy .pt dictionaries with: + own_indices: [N, K] padded with -1 + partner_indices: [N, K] padded with -1 + targets: [N] + +The native .ipd format also contains game IDs, allowing validation to split by +whole games instead of leaking neighboring positions across train/validation. +""" + +from __future__ import annotations + +import argparse +import struct +from pathlib import Path + +import numpy as np +import torch +from torch import nn +from torch.utils.data import DataLoader, Subset, TensorDataset, random_split + +from model import FEATURE_COUNT, PhoenixNet + +DATA_MAGIC = b"IPDATA1\0" +DATA_VERSION = 1 +DATA_HEADER = struct.Struct(" dict[str, torch.Tensor]: + with path.open("rb") as f: + magic = f.read(8) + if magic != DATA_MAGIC: + raise ValueError(f"{path} is not an IronPhoenix IPDATA1 dataset") + + raw_header = f.read(DATA_HEADER.size) + if len(raw_header) != DATA_HEADER.size: + raise ValueError("truncated IPD header") + + ( + version, + feature_count, + max_features, + teacher_depth, + score_scale, + seed, + record_count, + ) = DATA_HEADER.unpack(raw_header) + + if version != DATA_VERSION: + raise ValueError(f"unsupported IPD version {version}") + if feature_count != FEATURE_COUNT: + raise ValueError( + f"feature count mismatch: dataset={feature_count} model={FEATURE_COUNT}" + ) + if max_features <= 0: + raise ValueError("invalid max feature count") + + own = np.full((record_count, max_features), -1, dtype=np.int32) + partner = np.full((record_count, max_features), -1, dtype=np.int32) + targets = np.empty(record_count, dtype=np.float32) + teacher_cp = np.empty(record_count, dtype=np.int32) + game_ids = np.empty(record_count, dtype=np.int64) + plies = np.empty(record_count, dtype=np.int32) + sides = np.empty(record_count, dtype=np.int16) + flags = np.empty(record_count, dtype=np.uint8) + + for row in range(record_count): + raw_record = f.read(RECORD_HEADER.size) + if len(raw_record) != RECORD_HEADER.size: + raise ValueError(f"truncated record header at sample {row}") + + ( + own_count, + partner_count, + target, + cp, + game_id, + ply, + side, + record_flags, + ) = RECORD_HEADER.unpack(raw_record) + + if own_count > max_features or partner_count > max_features: + raise ValueError(f"feature count exceeds header limit at sample {row}") + + total_features = own_count + partner_count + raw_features = f.read(total_features * 2) + if len(raw_features) != total_features * 2: + raise ValueError(f"truncated feature payload at sample {row}") + + values = np.frombuffer(raw_features, dtype=" dict[str, torch.Tensor]: + if path.suffix.lower() == ".ipd": + return load_ipd(path) + + data = torch.load(path, map_location="cpu") + if not isinstance(data, dict): + raise ValueError(".pt dataset must be a dictionary") + + required = {"own_indices", "partner_indices", "targets"} + missing = required.difference(data) + if missing: + raise ValueError(f"dataset is missing keys: {sorted(missing)}") + + return data + + +def pack_embedding_bag(rows: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """Convert padded [B, K] sparse rows into EmbeddingBag indices/offsets.""" + mask = rows.ge(0) + counts = mask.sum(dim=1, dtype=torch.long) + flat = rows[mask].long() + + offsets = torch.zeros(rows.shape[0], dtype=torch.long) + if rows.shape[0] > 1: + offsets[1:] = torch.cumsum(counts[:-1], dim=0) + return flat, offsets + + +def split_dataset( + dataset: TensorDataset, + game_ids: torch.Tensor | None, + val_fraction: float, + seed: int, +) -> tuple[Subset | torch.utils.data.Dataset, Subset | torch.utils.data.Dataset]: + if game_ids is None: + val_size = max(1, int(len(dataset) * val_fraction)) + train_size = len(dataset) - val_size + generator = torch.Generator().manual_seed(seed) + return random_split(dataset, [train_size, val_size], generator=generator) + + unique_games = torch.unique(game_ids) + if unique_games.numel() < 2: + val_size = max(1, int(len(dataset) * val_fraction)) + train_size = len(dataset) - val_size + generator = torch.Generator().manual_seed(seed) + return random_split(dataset, [train_size, val_size], generator=generator) + + generator = torch.Generator().manual_seed(seed) + permutation = torch.randperm(unique_games.numel(), generator=generator) + unique_games = unique_games[permutation] + + val_game_count = max(1, int(unique_games.numel() * val_fraction)) + val_games = unique_games[:val_game_count] + val_mask = torch.isin(game_ids, val_games) + + val_indices = torch.nonzero(val_mask, as_tuple=False).flatten().tolist() + train_indices = torch.nonzero(~val_mask, as_tuple=False).flatten().tolist() + + if not train_indices or not val_indices: + val_size = max(1, int(len(dataset) * val_fraction)) + train_size = len(dataset) - val_size + return random_split(dataset, [train_size, val_size], generator=generator) + + print( + f"game split: {unique_games.numel():,} games, " + f"{len(train_indices):,} train positions, {len(val_indices):,} validation positions" + ) + return Subset(dataset, train_indices), Subset(dataset, val_indices) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("dataset", type=Path) + parser.add_argument("--epochs", type=int, default=20) + parser.add_argument("--batch-size", type=int, default=1024) + parser.add_argument("--lr", type=float, default=1e-3) + parser.add_argument("--val-fraction", type=float, default=0.05) + parser.add_argument("--split-seed", type=int, default=20261009) + parser.add_argument("--workers", type=int, default=0, help="DataLoader worker processes") + parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") + parser.add_argument("-o", "--output", type=Path, default=Path("phoenixnet-v1.pt")) + args = parser.parse_args() + + if not (0.0 < args.val_fraction < 1.0): + raise ValueError("--val-fraction must be between 0 and 1") + + data = load_dataset(args.dataset) + own = data["own_indices"] + partner = data["partner_indices"] + targets = data["targets"].float() + + if own.ndim != 2 or partner.ndim != 2 or targets.ndim != 1: + raise ValueError("dataset tensor shapes are invalid") + if own.shape[0] != partner.shape[0] or own.shape[0] != targets.shape[0]: + raise ValueError("dataset tensors have different sample counts") + if own.shape[0] < 2: + raise ValueError("dataset needs at least two samples") + + dataset = TensorDataset(own, partner, targets) + game_ids = data.get("game_ids") + if game_ids is not None: + game_ids = game_ids.long() + + train_set, val_set = split_dataset( + dataset, + game_ids, + args.val_fraction, + args.split_seed, + ) + + use_pin_memory = args.device.startswith("cuda") + train_loader = DataLoader( + train_set, + batch_size=args.batch_size, + shuffle=True, + num_workers=args.workers, + pin_memory=use_pin_memory, + ) + val_loader = DataLoader( + val_set, + batch_size=args.batch_size, + shuffle=False, + num_workers=args.workers, + pin_memory=use_pin_memory, + ) + + device = torch.device(args.device) + print(f"training on {device}") + + model = PhoenixNet().to(device) + optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr) + loss_fn = nn.SmoothL1Loss() + + best_val = float("inf") + + for epoch in range(1, args.epochs + 1): + model.train() + train_loss = 0.0 + train_items = 0 + + for own_rows, partner_rows, target in train_loader: + own_idx, own_offsets = pack_embedding_bag(own_rows) + partner_idx, partner_offsets = pack_embedding_bag(partner_rows) + + own_idx = own_idx.to(device, non_blocking=True) + own_offsets = own_offsets.to(device, non_blocking=True) + partner_idx = partner_idx.to(device, non_blocking=True) + partner_offsets = partner_offsets.to(device, non_blocking=True) + target = target.to(device, non_blocking=True) + + optimizer.zero_grad(set_to_none=True) + prediction = model(own_idx, own_offsets, partner_idx, partner_offsets) + loss = loss_fn(prediction, target) + loss.backward() + optimizer.step() + + train_loss += loss.item() * target.numel() + train_items += target.numel() + + model.eval() + val_loss = 0.0 + val_items = 0 + with torch.no_grad(): + for own_rows, partner_rows, target in val_loader: + own_idx, own_offsets = pack_embedding_bag(own_rows) + partner_idx, partner_offsets = pack_embedding_bag(partner_rows) + + prediction = model( + own_idx.to(device, non_blocking=True), + own_offsets.to(device, non_blocking=True), + partner_idx.to(device, non_blocking=True), + partner_offsets.to(device, non_blocking=True), + ) + target = target.to(device, non_blocking=True) + loss = loss_fn(prediction, target) + val_loss += loss.item() * target.numel() + val_items += target.numel() + + train_mean = train_loss / max(1, train_items) + val_mean = val_loss / max(1, val_items) + print(f"epoch={epoch:03d} train={train_mean:.6f} val={val_mean:.6f}") + + if val_mean < best_val: + best_val = val_mean + torch.save( + { + "model": model.state_dict(), + "val_loss": best_val, + "epoch": epoch, + "dataset": str(args.dataset), + }, + args.output, + ) + print(f"saved {args.output}") + + +if __name__ == "__main__": + main() diff --git a/src/eval.cpp b/src/eval.cpp index 3f2d673..6dddaa8 100644 --- a/src/eval.cpp +++ b/src/eval.cpp @@ -1,6 +1,7 @@ #include "ironphoenix/eval.hpp" #include "ironphoenix/geometry.hpp" +#include "ironphoenix/nnue.hpp" #include "ironphoenix/position.hpp" #include @@ -66,7 +67,7 @@ namespace ironphoenix::Eval { const int mobility = (attacks & ~blockedByTeam).popcount(); score += mobility * MOBILITY_WEIGHT[static_cast(pt)]; } - }; + }; scorePieces(KNIGHT); scorePieces(BISHOP); @@ -76,32 +77,36 @@ namespace ironphoenix::Eval { return score; } - int playerScore(const Position& pos, - Color c) noexcept { + int playerScore(const Position& pos, Color c) noexcept { return materialForColor(pos, c) + mobilityForColor(pos, c); } - } - - int evaluate(const Position& pos) noexcept { + int handcraftedEvaluate(const Position& pos) noexcept { + int team0 = 0; + int team1 = 0; - int team0 = 0; - int team1 = 0; + for (unsigned ci = 0; ci < COLOR_NB; ++ci) { + const Color c = static_cast(ci); + const int score = playerScore(pos, c); - for (unsigned ci = 0; ci < COLOR_NB; ++ci) { - const Color c = static_cast(ci); - const int score = playerScore(pos, c); + if (teamOf(c) == 0) + team0 += score; + else + team1 += score; + } - if (teamOf(c) == 0) - team0 += score; - else - team1 += score; + const int score = team0 - team1; + return teamOf(pos.sideToMove()) == 0 ? score : -score; } - const int score = team0 - team1; + } + + int evaluate(const Position& pos) noexcept { + if (NNUE::loaded()) + return NNUE::evaluate(pos); - return teamOf(pos.sideToMove()) == 0 ? score : -score; + return handcraftedEvaluate(pos); } } diff --git a/src/main.cpp b/src/main.cpp index cffe2c6..d643e56 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,10 +1,22 @@ +#include "ironphoenix/nnue.hpp" #include "ironphoenix/uci.hpp" +#include #include +#include int main() { - std::cerr - << "IronPhoenix | Built by love by Nick!\n"; + std::cerr << "IronPhoenix | Built by love by Nick!\n"; + + const char* envPath = std::getenv("IRONPHOENIX_NNUE"); + const std::string networkPath = envPath && *envPath + ? std::string(envPath) + : std::string("ironphoenix.nnue"); + + if (ironphoenix::NNUE::loadNetwork(networkPath)) + std::cerr << "PhoenixNet loaded: " << networkPath << '\n'; + else + std::cerr << "PhoenixNet not loaded; using HCE fallback\n"; ironphoenix::UciShell shell; return shell.run(std::cin, std::cout); diff --git a/src/multipv.cpp b/src/multipv.cpp new file mode 100644 index 0000000..e8548dc --- /dev/null +++ b/src/multipv.cpp @@ -0,0 +1,262 @@ +#include "ironphoenix/search.hpp" + +#include "ironphoenix/movegen.hpp" + +#include +#include +#include +#include +#include +#include + +namespace ironphoenix { +namespace { + +constexpr std::size_t ROOT_PV_CAPACITY = 128; + +struct RootLine final { + Move move{}; + int score = 0; + std::array pv{}; + int pvLength = 0; +}; + +} // namespace + +void SearchEngine::startMultiPV( + Position position, + SearchLimits limits, + int multiPV, + std::ostream& out) { + + if (multiPV <= 1) { + start(std::move(position), limits, out); + return; + } + + multiPV = std::clamp(multiPV, 2, MAX_MULTI_PV); + + stopAndWait(); + stopRequested_.store(false, std::memory_order_relaxed); + searching_.store(true, std::memory_order_relaxed); + tt_.newGeneration(); + + worker_ = std::thread([ + this, + position = std::move(position), + limits, + multiPV, + &out]() mutable { + + limits_ = limits; + runMultiPV(std::move(position), multiPV, out); + searching_.store(false, std::memory_order_relaxed); + }); +} + +void SearchEngine::emitMultiPVInfo( + std::ostream& out, + int depth, + int multiPvIndex, + int score, + const Move* pv, + int pvLength) { + + const std::int64_t ms = elapsedMs(); + const std::uint64_t nps = ms > 0 + ? nodes_ * 1000ull / static_cast(ms) + : nodes_ * 1000ull; + + std::ostringstream line; + line << "info depth " << depth + << " seldepth " << selDepth_ + << " multipv " << multiPvIndex; + + if (score >= MATE_THRESHOLD) { + line << " score mate " << (MATE_SCORE - score); + } + else if (score <= -MATE_THRESHOLD) { + line << " score mate -" << (MATE_SCORE + score); + } + else { + line << " score cp " << score; + } + + line << " nodes " << nodes_ + << " nps " << nps + << " hashfull " << tt_.hashfull() + << " time " << ms + << " pv"; + + for (int i = 0; i < pvLength; ++i) { + if (!pv[i]) + break; + line << ' ' << moveToString(pv[i]); + } + + std::lock_guard lock(outputMutex_); + out << line.str() << '\n'; + out.flush(); +} + +void SearchEngine::runMultiPV(Position position, int multiPV, std::ostream& out) { + startTime_ = std::chrono::steady_clock::now(); + timeBudgetMs_ = calculateTimeBudget(position, limits_); + nodes_ = 0; + selDepth_ = 0; + + for (auto& row : pv_) + row.fill(Move{}); + pvLength_.fill(0); + searchStack_.fill(HistoryContext{}); + + MoveList rootLegal; + generateLegalMoves(position, rootLegal); + + if (rootLegal.size == 0) { + emitBestMove(out, Move{}); + return; + } + + // Initial move ordering still benefits from TT/history/check/capture ordering. + orderMoves(position, rootLegal.begin(), rootLegal.end(), Move{}, 0); + + const int requestedLines = std::min(multiPV, static_cast(rootLegal.size)); + const int requestedDepth = limits_.depth > 0 + ? std::min(limits_.depth, MAX_PLY - 1) + : 0; + + Move bestMove = rootLegal.moves[0]; + std::vector lastComplete; + int depth = 1; + + while (!shouldStop()) { + if (requestedDepth > 0 && depth > requestedDepth) + break; + + if (!limits_.infinite + && requestedDepth == 0 + && limits_.nodes == 0 + && limits_.movetimeMs == 0 + && !limits_.hasClock + && depth > 8) { + break; + } + + // Previous-iteration score ordering is especially important for + // MultiPV because every legal root move receives a full-window search. + if (!lastComplete.empty()) { + for (std::size_t i = 0; i < lastComplete.size() && i < rootLegal.size; ++i) + rootLegal.moves[i] = lastComplete[i].move; + } + + selDepth_ = 0; + std::vector current; + current.reserve(rootLegal.size); + bool completedDepth = true; + + for (Move move : rootLegal) { + if (shouldStop()) { + completedDepth = false; + break; + } + + RootLine line; + line.move = move; + line.pv[0] = move; + line.pvLength = 1; + + if (isTerminalKingCapture(position, move)) { + line.score = MATE_SCORE - 1; + } + else { + const Color mover = position.sideToMove(); + StateInfo state; + position.makeMove(move, state); + + // rootLegal already contains only legal moves, but keeping this + // guard protects MultiPV if move generation rules evolve. + if (position.inCheck(mover)) { + position.undoMove(move, state); + continue; + } + + searchStack_[1] = HistoryContext{ + move, + position.pieceAt(move.to()), + mover + }; + + pvLength_[1] = 1; + line.score = -negamax( + position, + depth - 1, + -INF, + INF, + 1, + true + ); + + if (!shouldStop()) { + const int childLength = std::clamp(pvLength_[1], 1, MAX_PLY); + for (int i = 1; i < childLength; ++i) + line.pv[static_cast(i)] = pv_[1][i]; + line.pvLength = childLength; + } + + position.undoMove(move, state); + + if (shouldStop()) { + completedDepth = false; + break; + } + } + + current.push_back(line); + } + + if (!completedDepth || current.size() != rootLegal.size) + break; + + std::stable_sort( + current.begin(), + current.end(), + [](const RootLine& a, const RootLine& b) { + if (a.score != b.score) + return a.score > b.score; + return a.move.v < b.move.v; + }); + + lastComplete = current; + bestMove = lastComplete.front().move; + + const int lineCount = std::min(requestedLines, static_cast(lastComplete.size())); + for (int i = 0; i < lineCount; ++i) { + const RootLine& line = lastComplete[static_cast(i)]; + emitMultiPVInfo( + out, + depth, + i + 1, + line.score, + line.pv.data(), + line.pvLength + ); + } + + if (requestedDepth > 0 && depth == requestedDepth) + break; + if (limits_.nodes > 0 && nodes_ >= limits_.nodes) + break; + if (timeBudgetMs_ > 0 && elapsedMs() >= timeBudgetMs_) + break; + + if (depth < MAX_PLY - 1) + ++depth; + else if (!limits_.infinite) + break; + } + + emitBestMove(out, bestMove); +} + +} // namespace ironphoenix diff --git a/src/nnue.cpp b/src/nnue.cpp new file mode 100644 index 0000000..29cd615 --- /dev/null +++ b/src/nnue.cpp @@ -0,0 +1,299 @@ +#include "ironphoenix/nnue.hpp" + +#include "ironphoenix/geometry.hpp" +#include "ironphoenix/position.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ironphoenix::NNUE { +namespace { + +constexpr std::array MAGIC = {'I', 'P', 'N', 'N', 'U', 'E', '1', '\0'}; +constexpr std::uint32_t VERSION = 1; +constexpr int MAX_STATIC_EVAL = 28000; + +struct Network final { + float outputScale = 400.0f; + + // Shared sparse feature transformer. The same table is used for the + // current player's king and the partner king accumulator streams. + std::vector ftWeights; + std::array ftBias{}; + + // [own 128 | partner 128] -> 32 -> 32 -> 1 + std::array hidden1Weights{}; + std::array hidden1Bias{}; + std::array hidden2Weights{}; + std::array hidden2Bias{}; + std::array outputWeights{}; + float outputBias = 0.0f; +}; + +Network gNetwork; +bool gLoaded = false; +std::string gLoadedPath; + +IRONPHOENIX_FORCE_INLINE float crelu(float x) noexcept { + return std::clamp(x, 0.0f, 1.0f); +} + +template +bool readExact(std::ifstream& in, T& value) noexcept { + return static_cast(in.read(reinterpret_cast(&value), sizeof(T))); +} + +template +bool readArray(std::ifstream& in, std::array& values) noexcept { + return static_cast(in.read( + reinterpret_cast(values.data()), + static_cast(sizeof(T) * N))); +} + +bool readVector(std::ifstream& in, std::vector& values) noexcept { + return static_cast(in.read( + reinterpret_cast(values.data()), + static_cast(sizeof(float) * values.size()))); +} + +void refreshAccumulator( + const Position& pos, + Color perspective, + Square anchorKing, + std::array& accumulator) noexcept { + + accumulator = gNetwork.ftBias; + + for (unsigned ci = 0; ci < COLOR_NB; ++ci) { + const Color pieceColor = static_cast(ci); + + for (unsigned pti = PAWN; pti <= KING; ++pti) { + const PieceType pt = static_cast(pti); + Bitboard pieces = pos.pieces(pieceColor, pt); + + while (pieces) { + const Square sq = popLsb(pieces); + const Piece piece = makePiece(pieceColor, pt); + const std::size_t feature = featureIndex(perspective, anchorKing, piece, sq); + const std::size_t base = feature * FT_SIZE; + + for (std::size_t i = 0; i < FT_SIZE; ++i) + accumulator[i] += gNetwork.ftWeights[base + i]; + } + } + } + + for (float& value : accumulator) + value = crelu(value); +} + +} // namespace + +Square canonicalSquare(Color perspective, Square sq) noexcept { + if (sq == SQ_NONE || sq >= SQUARE_NB) + return SQ_NONE; + + const int file = static_cast(Geometry::fileOf(sq)); + const int rank = static_cast(Geometry::rankOf(sq)); + + int canonicalFile = file; + int canonicalRank = rank; + + switch (perspective) { + case RED: + break; + case BLUE: + canonicalFile = 13 - rank; + canonicalRank = file; + break; + case YELLOW: + canonicalFile = 13 - file; + canonicalRank = 13 - rank; + break; + case GREEN: + canonicalFile = rank; + canonicalRank = 13 - file; + break; + } + + return Geometry::square(canonicalFile, canonicalRank); +} + +unsigned relativeColor(Color perspective, Color pieceColor) noexcept { + return (static_cast(pieceColor) + COLOR_NB + - static_cast(perspective)) & 3u; +} + +unsigned kingBucket(Color perspective, Square kingSq) noexcept { + if (kingSq == SQ_NONE || kingSq >= SQUARE_NB) + return 16u; + + const Square canonical = canonicalSquare(perspective, kingSq); + if (canonical == SQ_NONE) + return 16u; + + const unsigned file = Geometry::fileOf(canonical); + const unsigned rank = Geometry::rankOf(canonical); + + // Four coarse regions on each axis. 14 does not divide evenly by four, + // so multiply before dividing to keep the buckets balanced. + const unsigned fileBucket = std::min(3u, (file * 4u) / BOARD_FILES); + const unsigned rankBucket = std::min(3u, (rank * 4u) / BOARD_RANKS); + return rankBucket * 4u + fileBucket; +} + +std::size_t featureIndex( + Color perspective, + Square anchorKing, + Piece piece, + Square pieceSq) noexcept { + + if (piece == NO_PIECE || pieceSq == SQ_NONE || pieceSq >= SQUARE_NB) + return 0; + + const PieceType pt = typeOf(piece); + if (pt < PAWN || pt > KING) + return 0; + + const Square canonical = canonicalSquare(perspective, pieceSq); + if (canonical == SQ_NONE) + return 0; + + const std::size_t bucket = kingBucket(perspective, anchorKing); + const std::size_t relColor = relativeColor(perspective, colorOf(piece)); + const std::size_t pieceType = static_cast(pt) - 1u; + + return (((bucket * RELATIVE_COLORS + relColor) * NNUE_PIECE_TYPES + pieceType) + * SQUARE_NB) + canonical; +} + +bool loadNetwork(const std::string& path) noexcept { + try { + std::ifstream in(path, std::ios::binary); + if (!in) + return false; + + std::array magic{}; + if (!in.read(magic.data(), static_cast(magic.size())) || magic != MAGIC) + return false; + + std::uint32_t version = 0; + std::uint32_t featureCount = 0; + std::uint32_t ftSize = 0; + std::uint32_t hidden1Size = 0; + std::uint32_t hidden2Size = 0; + float outputScale = 0.0f; + + if (!readExact(in, version) + || !readExact(in, featureCount) + || !readExact(in, ftSize) + || !readExact(in, hidden1Size) + || !readExact(in, hidden2Size) + || !readExact(in, outputScale)) + return false; + + if (version != VERSION + || featureCount != FEATURE_COUNT + || ftSize != FT_SIZE + || hidden1Size != HIDDEN1_SIZE + || hidden2Size != HIDDEN2_SIZE + || !std::isfinite(outputScale) + || outputScale <= 0.0f) + return false; + + Network candidate; + candidate.outputScale = outputScale; + candidate.ftWeights.resize(FEATURE_COUNT * FT_SIZE); + + if (!readVector(in, candidate.ftWeights) + || !readArray(in, candidate.ftBias) + || !readArray(in, candidate.hidden1Weights) + || !readArray(in, candidate.hidden1Bias) + || !readArray(in, candidate.hidden2Weights) + || !readArray(in, candidate.hidden2Bias) + || !readArray(in, candidate.outputWeights) + || !readExact(in, candidate.outputBias)) + return false; + + // Reject malformed files with trailing data. This catches most + // architecture/export mismatches instead of silently evaluating junk. + char trailing = 0; + if (in.read(&trailing, 1)) + return false; + + gNetwork = std::move(candidate); + gLoadedPath = path; + gLoaded = true; + return true; + } + catch (...) { + return false; + } +} + +void unloadNetwork() noexcept { + gNetwork = Network{}; + gLoadedPath.clear(); + gLoaded = false; +} + +bool loaded() noexcept { + return gLoaded; +} + +const std::string& loadedPath() noexcept { + return gLoadedPath; +} + +int evaluate(const Position& pos) noexcept { + if (!gLoaded) + return 0; + + const Color perspective = pos.sideToMove(); + const Color partner = static_cast(static_cast(perspective) ^ 2u); + + std::array own{}; + std::array partnerAcc{}; + refreshAccumulator(pos, perspective, pos.kingSquare(perspective), own); + refreshAccumulator(pos, perspective, pos.kingSquare(partner), partnerAcc); + + std::array input{}; + std::copy(own.begin(), own.end(), input.begin()); + std::copy(partnerAcc.begin(), partnerAcc.end(), input.begin() + FT_SIZE); + + std::array hidden1{}; + for (std::size_t out = 0; out < HIDDEN1_SIZE; ++out) { + float sum = gNetwork.hidden1Bias[out]; + const std::size_t row = out * input.size(); + for (std::size_t in = 0; in < input.size(); ++in) + sum += gNetwork.hidden1Weights[row + in] * input[in]; + hidden1[out] = crelu(sum); + } + + std::array hidden2{}; + for (std::size_t out = 0; out < HIDDEN2_SIZE; ++out) { + float sum = gNetwork.hidden2Bias[out]; + const std::size_t row = out * HIDDEN1_SIZE; + for (std::size_t in = 0; in < HIDDEN1_SIZE; ++in) + sum += gNetwork.hidden2Weights[row + in] * hidden1[in]; + hidden2[out] = crelu(sum); + } + + float output = gNetwork.outputBias; + for (std::size_t i = 0; i < HIDDEN2_SIZE; ++i) + output += gNetwork.outputWeights[i] * hidden2[i]; + + if (!std::isfinite(output)) + return 0; + + const float cp = output * gNetwork.outputScale; + return std::clamp(static_cast(std::lround(cp)), -MAX_STATIC_EVAL, MAX_STATIC_EVAL); +} + +} // namespace ironphoenix::NNUE diff --git a/src/uci.cpp b/src/uci.cpp index 666d18f..c8d8ce4 100644 --- a/src/uci.cpp +++ b/src/uci.cpp @@ -234,6 +234,7 @@ namespace ironphoenix { " ucinewgame - reload current setup start position\n" " setoption name Setup value \n" " setoption name StartFEN value \n" + " setoption name MultiPV value <1..32>\n" " position fen [moves ...]\n" " position startpos [moves ...] (Modern is built in by default)\n" " d - display board\n" @@ -332,6 +333,20 @@ namespace ironphoenix { return true; } + constexpr std::string_view multiPvPrefix = "name MultiPV value "; + if (startsWith(args, multiPvPrefix)) { + std::int64_t value = 0; + if (!parseNonNegative64(trim(args.substr(multiPvPrefix.size())), value) + || value < 1 || value > SearchEngine::MAX_MULTI_PV) { + out << "info string MultiPV must be between 1 and " + << SearchEngine::MAX_MULTI_PV << '\n'; + return false; + } + multiPV_ = static_cast(value); + out << "info string MultiPV set to " << multiPV_ << '\n'; + return true; + } + if (args == "name Clear Hash") { search_.clearHash(); out << "info string Hash cleared\n"; @@ -508,7 +523,10 @@ namespace ironphoenix { } } - search_.start(pos_, limits, out); + if (multiPV_ > 1) + search_.startMultiPV(pos_, limits, multiPV_, out); + else + search_.start(pos_, limits, out); return true; } @@ -523,6 +541,7 @@ namespace ironphoenix { << "option name Setup type combo default Modern var Modern var Classic var BY var BYG var RG var Custom\n" << "option name StartFEN type string default \n" << "option name Hash type spin default 64 min 1 max 4096\n" + << "option name MultiPV type spin default 1 min 1 max " << SearchEngine::MAX_MULTI_PV << "\n" << "option name Clear Hash type button\n" << "uciok\n"; } diff --git a/tools/nnue_dataset.cpp b/tools/nnue_dataset.cpp new file mode 100644 index 0000000..772fcb1 --- /dev/null +++ b/tools/nnue_dataset.cpp @@ -0,0 +1,643 @@ +#include "ironphoenix/fen.hpp" +#include "ironphoenix/movegen.hpp" +#include "ironphoenix/nnue.hpp" +#include "ironphoenix/search.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ironphoenix { +namespace { + +constexpr std::array DATA_MAGIC = {'I', 'P', 'D', 'A', 'T', 'A', '1', '\0'}; +constexpr std::uint32_t DATA_VERSION = 1; +constexpr std::uint32_t MAX_FEATURES_PER_STREAM = 64; + +constexpr std::string_view MODERN_START_FEN = + "R-0,0,0,0-1,1,1,1-1,1,1,1-0,0,0,0-0-" + "x,x,x,yR,yN,yB,yK,yQ,yB,yN,yR,x,x,x/" + "x,x,x,yP,yP,yP,yP,yP,yP,yP,yP,x,x,x/" + "x,x,x,8,x,x,x/" + "bR,bP,10,gP,gR/" + "bN,bP,10,gP,gN/" + "bB,bP,10,gP,gB/" + "bQ,bP,10,gP,gK/" + "bK,bP,10,gP,gQ/" + "bB,bP,10,gP,gB/" + "bN,bP,10,gP,gN/" + "bR,bP,10,gP,gR/" + "x,x,x,8,x,x,x/" + "x,x,x,rP,rP,rP,rP,rP,rP,rP,rP,x,x,x/" + "x,x,x,rR,rN,rB,rQ,rK,rB,rN,rR,x,x,x"; + +struct Options final { + std::string output = "phoenix_dataset.ipd"; + std::uint64_t positions = 500000; + int depth = 8; + int randomPlies = 4; + int skipPlies = 8; + int sampleEvery = 3; + int maxPlies = 320; + int cpClamp = 4000; + float scoreScale = 400.0f; + std::size_t hashMB = 16; + unsigned workers = 1; + std::uint64_t seed = 0x49524f4e50484f45ULL; +}; + +struct TeacherResult final { + bool ok = false; + bool mate = false; + int cp = 0; + Move bestMove{}; +}; + +struct Record final { + std::vector own; + std::vector partner; + float target = 0.0f; + std::int32_t teacherCp = 0; + std::uint32_t gameId = 0; + std::uint16_t ply = 0; + std::uint8_t sideToMove = 0; + std::uint8_t flags = 0; +}; + +template +void writeValue(std::ofstream& out, const T& value) { + out.write(reinterpret_cast(&value), sizeof(T)); +} + +class DatasetWriter final { +public: + explicit DatasetWriter(const Options& options) + : out_(options.output, std::ios::binary | std::ios::trunc) { + if (!out_) + return; + + out_.write(DATA_MAGIC.data(), static_cast(DATA_MAGIC.size())); + writeValue(out_, DATA_VERSION); + writeValue(out_, static_cast(NNUE::FEATURE_COUNT)); + writeValue(out_, MAX_FEATURES_PER_STREAM); + writeValue(out_, static_cast(options.depth)); + writeValue(out_, options.scoreScale); + writeValue(out_, options.seed); + + countPosition_ = out_.tellp(); + const std::uint64_t zero = 0; + writeValue(out_, zero); + } + + ~DatasetWriter() { + finalize(); + } + + [[nodiscard]] bool good() const noexcept { + return static_cast(out_); + } + + [[nodiscard]] std::uint64_t count() const noexcept { + return reserved_.load(std::memory_order_relaxed); + } + + bool tryWrite(const Record& record, std::uint64_t limit) { + if (record.own.size() > MAX_FEATURES_PER_STREAM + || record.partner.size() > MAX_FEATURES_PER_STREAM) + return false; + + std::uint64_t current = reserved_.load(std::memory_order_relaxed); + while (current < limit) { + if (reserved_.compare_exchange_weak( + current, + current + 1, + std::memory_order_relaxed, + std::memory_order_relaxed)) + break; + } + if (current >= limit) + return true; + + std::lock_guard lock(mutex_); + if (!out_ || finalized_) + return false; + + const auto ownCount = static_cast(record.own.size()); + const auto partnerCount = static_cast(record.partner.size()); + + writeValue(out_, ownCount); + writeValue(out_, partnerCount); + writeValue(out_, record.target); + writeValue(out_, record.teacherCp); + writeValue(out_, record.gameId); + writeValue(out_, record.ply); + writeValue(out_, record.sideToMove); + writeValue(out_, record.flags); + + if (!record.own.empty()) { + out_.write( + reinterpret_cast(record.own.data()), + static_cast(record.own.size() * sizeof(std::uint16_t))); + } + if (!record.partner.empty()) { + out_.write( + reinterpret_cast(record.partner.data()), + static_cast(record.partner.size() * sizeof(std::uint16_t))); + } + + return static_cast(out_); + } + + void finalize() { + std::lock_guard lock(mutex_); + if (finalized_ || !out_) + return; + + const auto end = out_.tellp(); + out_.seekp(countPosition_); + const std::uint64_t recordCount = count(); + writeValue(out_, recordCount); + out_.seekp(end); + out_.flush(); + finalized_ = true; + } + +private: + std::ofstream out_; + std::streampos countPosition_{}; + std::atomic reserved_{0}; + std::mutex mutex_; + bool finalized_ = false; +}; + +Direction kingSideDirection(Color c) noexcept { + switch (c) { + case RED: return EAST; + case BLUE: return SOUTH; + case YELLOW: return WEST; + case GREEN: return NORTH; + } + return EAST; +} + +Square findCastlingRook(const Position& pos, Color c, Square king, Direction direction) noexcept { + for (Square sq = Geometry::step(king, direction); + sq != SQ_NONE; + sq = Geometry::step(sq, direction)) { + if (pos.pieceAt(sq) == makePiece(c, ROOK)) + return sq; + } + return SQ_NONE; +} + +void configureStandardCastling(Position& pos) noexcept { + for (unsigned ci = 0; ci < COLOR_NB; ++ci) { + const Color c = static_cast(ci); + const Square king = pos.kingSquare(c); + if (king == SQ_NONE) + continue; + + for (unsigned laneIndex = 0; laneIndex < 2; ++laneIndex) { + Direction direction = kingSideDirection(c); + if (laneIndex == 1) + direction = Geometry::opposite(direction); + + const Square rook = findCastlingRook(pos, c, king, direction); + const Square rookTo = Geometry::step(king, direction); + const Square kingTo = rookTo == SQ_NONE + ? SQ_NONE + : Geometry::step(rookTo, direction); + + CastleLane lane{}; + if (rook != SQ_NONE && rookTo != SQ_NONE && kingTo != SQ_NONE && kingTo != rook) { + lane.kingFrom = king; + lane.rookFrom = rook; + lane.kingTo = kingTo; + lane.rookTo = rookTo; + lane.rightBit = static_cast(2u * ci + laneIndex); + } + pos.configureCastle(c, laneIndex, lane); + } + } +} + +bool resetModern(Position& pos, Fen4State& state, std::string& error) { + pos.setRulesetId(1); + if (!setFromFen4(pos, MODERN_START_FEN, state, error)) + return false; + configureStandardCastling(pos); + return true; +} + +void collectFeatures( + const Position& pos, + Color perspective, + Square anchorKing, + std::vector& output) { + + output.clear(); + output.reserve(MAX_FEATURES_PER_STREAM); + + for (unsigned ci = 0; ci < COLOR_NB; ++ci) { + const Color pieceColor = static_cast(ci); + for (unsigned pti = PAWN; pti <= KING; ++pti) { + const PieceType pt = static_cast(pti); + Bitboard pieces = pos.pieces(pieceColor, pt); + + while (pieces) { + const Square sq = popLsb(pieces); + const std::size_t feature = NNUE::featureIndex( + perspective, + anchorKing, + makePiece(pieceColor, pt), + sq); + + if (feature < NNUE::FEATURE_COUNT + && feature <= std::numeric_limits::max()) { + output.push_back(static_cast(feature)); + } + } + } + } +} + +TeacherResult searchPosition(SearchEngine& search, Position& pos, int depth) { + SearchLimits limits; + limits.depth = depth; + + std::ostringstream output; + search.start(pos, limits, output); + while (search.searching()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + search.stopAndWait(); + + enum class ScoreKind { None, Cp, Mate }; + ScoreKind scoreKind = ScoreKind::None; + int cp = 0; + std::string bestMoveText; + + std::istringstream lines(output.str()); + std::string line; + while (std::getline(lines, line)) { + if (line.rfind("info ", 0) == 0) { + constexpr std::string_view CP_TOKEN = " score cp "; + constexpr std::string_view MATE_TOKEN = " score mate "; + const auto cpPos = line.find(CP_TOKEN); + const auto matePos = line.find(MATE_TOKEN); + + if (cpPos != std::string::npos) { + try { + cp = std::stoi(line.substr(cpPos + CP_TOKEN.size())); + scoreKind = ScoreKind::Cp; + } + catch (...) { + scoreKind = ScoreKind::None; + } + } + else if (matePos != std::string::npos) { + scoreKind = ScoreKind::Mate; + } + } + else if (line.rfind("bestmove ", 0) == 0) { + bestMoveText = line.substr(9); + } + } + + TeacherResult result; + if (bestMoveText.empty() || bestMoveText == "0000" || scoreKind == ScoreKind::None) + return result; + + std::string error; + if (!parseLegalMove(pos, bestMoveText, result.bestMove, error)) + return result; + + result.ok = true; + result.mate = scoreKind == ScoreKind::Mate; + result.cp = cp; + return result; +} + +Move chooseRandomOpeningMove(Position& pos, std::mt19937_64& rng) { + MoveList legal; + generateLegalMoves(pos, legal); + if (legal.size == 0) + return {}; + + std::array candidates{}; + std::size_t candidateCount = 0; + for (Move move : legal) { + if (!isTerminalKingCapture(pos, move)) + candidates[candidateCount++] = move; + } + + if (candidateCount == 0) + return legal.moves[0]; + + std::uniform_int_distribution pick(0, candidateCount - 1); + return candidates[pick(rng)]; +} + +void workerMain( + unsigned workerId, + const Options& options, + DatasetWriter& writer, + std::atomic& nextGameId, + std::atomic& failed) { + + SearchEngine search; + if (!search.setHashSizeMB(options.hashMB)) { + std::cerr << "worker " << workerId << ": unable to allocate hash\n"; + failed.store(true, std::memory_order_relaxed); + return; + } + + std::mt19937_64 rng( + options.seed + 0x9E3779B97F4A7C15ULL * (static_cast(workerId) + 1ULL)); + + Position pos; + Fen4State fenState; + std::string error; + + while (!failed.load(std::memory_order_relaxed) && writer.count() < options.positions) { + const std::uint32_t gameId = nextGameId.fetch_add(1, std::memory_order_relaxed); + + if (!resetModern(pos, fenState, error)) { + std::cerr << "worker " << workerId << ": start position failed: " << error << '\n'; + failed.store(true, std::memory_order_relaxed); + return; + } + search.newGame(); + + for (int ply = 0; ply < options.maxPlies; ++ply) { + if (failed.load(std::memory_order_relaxed) || writer.count() >= options.positions) + break; + + MoveList legal; + generateLegalMoves(pos, legal); + if (legal.size == 0) + break; + + Move move; + TeacherResult teacher; + + if (ply < options.randomPlies) { + move = chooseRandomOpeningMove(pos, rng); + } + else { + teacher = searchPosition(search, pos, options.depth); + if (!teacher.ok) + break; + + move = teacher.bestMove; + + const bool samplePly = ply >= options.skipPlies + && ((ply - options.skipPlies) % options.sampleEvery == 0); + + if (samplePly && !teacher.mate) { + const int clippedCp = std::clamp(teacher.cp, -options.cpClamp, options.cpClamp); + + Record record; + record.target = static_cast(clippedCp) / options.scoreScale; + record.teacherCp = teacher.cp; + record.gameId = gameId; + record.ply = static_cast(std::min(ply, 65535)); + record.sideToMove = static_cast(pos.sideToMove()); + + if (pos.inCheck()) + record.flags |= 1u; + if (pos.aliveMask() != 0xF) + record.flags |= 2u; + + const Color perspective = pos.sideToMove(); + const Color partner = static_cast(static_cast(perspective) ^ 2u); + collectFeatures(pos, perspective, pos.kingSquare(perspective), record.own); + collectFeatures(pos, perspective, pos.kingSquare(partner), record.partner); + + if (record.own.size() != record.partner.size()) { + std::cerr << "worker " << workerId << ": feature stream size mismatch\n"; + failed.store(true, std::memory_order_relaxed); + return; + } + + if (!writer.tryWrite(record, options.positions)) { + std::cerr << "worker " << workerId << ": dataset write failed\n"; + failed.store(true, std::memory_order_relaxed); + return; + } + + const auto count = writer.count(); + if (count % 1000 == 0 || count == options.positions) { + std::cout << "positions " << count << '/' << options.positions + << " games " << (gameId + 1) + << " worker " << workerId << '\n'; + } + } + } + + if (!move) + break; + + // Chess.com-style Teams ends immediately when an enemy king is + // captured. Never generate post-terminal positions for training. + const bool terminalKingCapture = isTerminalKingCapture(pos, move); + + StateInfo state; + pos.makeMove(move, state); + + if (terminalKingCapture) + break; + } + } +} + +bool parseUnsigned(std::string_view text, std::uint64_t& value) { + try { + std::size_t used = 0; + value = std::stoull(std::string(text), &used); + return used == text.size(); + } + catch (...) { + return false; + } +} + +bool parseSigned(std::string_view text, int& value) { + try { + std::size_t used = 0; + value = std::stoi(std::string(text), &used); + return used == text.size(); + } + catch (...) { + return false; + } +} + +void printUsage() { + std::cout + << "IronPhoenix PhoenixNet dataset generator\n\n" + << "Usage: ironphoenix_dataset [options]\n\n" + << " --output Output .ipd file (default phoenix_dataset.ipd)\n" + << " --positions Number of samples (default 500000)\n" + << " --depth Teacher search depth (default 8)\n" + << " --workers Parallel self-play workers (default 1)\n" + << " --hash Hash MB per worker (default 16)\n" + << " --random-plies Random opening plies (default 4)\n" + << " --skip-plies Earliest saved ply (default 8)\n" + << " --sample-every Save one of every N plies (default 3)\n" + << " --max-plies Maximum plies per game (default 320)\n" + << " --cp-clamp Teacher CP clamp (default 4000)\n" + << " --score-scale CP per target unit (default 400)\n" + << " --seed RNG seed\n" + << " --help Show this help\n"; +} + +bool parseOptions(int argc, char** argv, Options& options, bool& showedHelp) { + showedHelp = false; + + for (int i = 1; i < argc; ++i) { + const std::string_view arg = argv[i]; + auto value = [&]() -> std::string_view { + if (i + 1 >= argc) + return {}; + return argv[++i]; + }; + + if (arg == "--help" || arg == "-h") { + printUsage(); + showedHelp = true; + return false; + } + else if (arg == "--output") { + const auto text = value(); + if (text.empty()) return false; + options.output = text; + } + else if (arg == "--positions") { + std::uint64_t parsed = 0; + if (!parseUnsigned(value(), parsed) || parsed == 0) return false; + options.positions = parsed; + } + else if (arg == "--depth") { + if (!parseSigned(value(), options.depth) || options.depth < 1 || options.depth > 32) return false; + } + else if (arg == "--workers") { + std::uint64_t parsed = 0; + if (!parseUnsigned(value(), parsed) || parsed == 0 || parsed > 128) return false; + options.workers = static_cast(parsed); + } + else if (arg == "--hash") { + std::uint64_t parsed = 0; + if (!parseUnsigned(value(), parsed) || parsed == 0 || parsed > 4096) return false; + options.hashMB = static_cast(parsed); + } + else if (arg == "--random-plies") { + if (!parseSigned(value(), options.randomPlies) || options.randomPlies < 0) return false; + } + else if (arg == "--skip-plies") { + if (!parseSigned(value(), options.skipPlies) || options.skipPlies < 0) return false; + } + else if (arg == "--sample-every") { + if (!parseSigned(value(), options.sampleEvery) || options.sampleEvery < 1) return false; + } + else if (arg == "--max-plies") { + if (!parseSigned(value(), options.maxPlies) || options.maxPlies < 1) return false; + } + else if (arg == "--cp-clamp") { + if (!parseSigned(value(), options.cpClamp) || options.cpClamp < 1 || options.cpClamp >= 29000) return false; + } + else if (arg == "--score-scale") { + try { + options.scoreScale = std::stof(std::string(value())); + } + catch (...) { + return false; + } + if (!(options.scoreScale > 0.0f)) return false; + } + else if (arg == "--seed") { + if (!parseUnsigned(value(), options.seed)) return false; + } + else { + std::cerr << "unknown option: " << arg << '\n'; + return false; + } + } + + if (options.skipPlies < options.randomPlies) + options.skipPlies = options.randomPlies; + return true; +} + +} // namespace +} // namespace ironphoenix + +int main(int argc, char** argv) { + using namespace ironphoenix; + + Options options; + bool showedHelp = false; + if (!parseOptions(argc, argv, options, showedHelp)) { + if (!showedHelp) + printUsage(); + return showedHelp ? 0 : 1; + } + + DatasetWriter writer(options); + if (!writer.good()) { + std::cerr << "unable to open output file: " << options.output << '\n'; + return 1; + } + + std::cout + << "PhoenixNet dataset generation\n" + << "output " << options.output << '\n' + << "positions " << options.positions << '\n' + << "depth " << options.depth << '\n' + << "workers " << options.workers << '\n' + << "hash/worker " << options.hashMB << " MB\n" + << "random plies " << options.randomPlies << '\n' + << "skip plies " << options.skipPlies << '\n' + << "sample every " << options.sampleEvery << '\n' + << "seed " << options.seed << "\n\n"; + + std::atomic nextGameId{0}; + std::atomic failed{false}; + std::vector workers; + workers.reserve(options.workers); + + for (unsigned workerId = 0; workerId < options.workers; ++workerId) { + workers.emplace_back( + workerMain, + workerId, + std::cref(options), + std::ref(writer), + std::ref(nextGameId), + std::ref(failed)); + } + + for (auto& worker : workers) + worker.join(); + + writer.finalize(); + + if (failed.load(std::memory_order_relaxed)) { + std::cerr << "dataset generation failed\n"; + return 1; + } + + std::cout << "done: wrote " << writer.count() + << " positions to " << options.output << '\n'; + return 0; +} diff --git a/tools/nnue_dataset_resumable.cpp b/tools/nnue_dataset_resumable.cpp new file mode 100644 index 0000000..8ff2a0a --- /dev/null +++ b/tools/nnue_dataset_resumable.cpp @@ -0,0 +1,700 @@ +// Graceful/resumable PhoenixNet dataset generator entry point. +// +// Reuse the established generator implementation while replacing only the +// orchestration/search loop pieces needed for Ctrl+C, --resume, recovery, and +// denser progress reporting. + +#define main ironphoenix_dataset_legacy_main +#define workerMain legacyWorkerMain +#define searchPosition legacySearchPosition +#include "nnue_dataset_with_append.cpp" +#undef searchPosition +#undef workerMain +#undef main + +#include + +namespace ironphoenix { +namespace { + +constexpr int EXPLORATION_MAX_PLY = 96; + +volatile std::sig_atomic_t gStopRequested = 0; +std::atomic gLastReported{0}; +std::mutex gProgressMutex; + +void handleInterrupt(int) noexcept { + gStopRequested = 1; +} + +[[nodiscard]] bool stopRequested() noexcept { + return gStopRequested != 0; +} + +void reportProgress( + const DatasetWriter& writer, + const GenerationStats& stats, + unsigned workerId) { + + const std::uint64_t count = writer.addedCount(); + if (count == 0) + return; + + const std::uint64_t milestone = (count / 10ULL) * 10ULL; + const bool finalCount = count >= writer.targetAdded(); + const std::uint64_t wanted = finalCount ? writer.targetAdded() : milestone; + if (wanted == 0) + return; + + std::uint64_t previous = gLastReported.load(std::memory_order_relaxed); + while (wanted > previous) { + if (gLastReported.compare_exchange_weak( + previous, + wanted, + std::memory_order_relaxed, + std::memory_order_relaxed)) { + + const std::uint64_t currentAdded = writer.addedCount(); + const std::uint64_t baseCount = writer.totalCount() - currentAdded; + + std::lock_guard lock(gProgressMutex); + std::cout << "new positions " << wanted << '/' << writer.targetAdded() + << " total " << (baseCount + wanted) + << " duplicates " << stats.duplicateSamples.load(std::memory_order_relaxed) + << " explored " << stats.exploredMoves.load(std::memory_order_relaxed) + << " worker " << workerId << '\n'; + std::cout.flush(); + return; + } + } +} + +bool recoverTrailingRecords( + const std::filesystem::path& path, + DatasetScan& scan, + DatasetDeduper* deduper, + std::uint64_t& recovered, + std::string& error) { + + recovered = 0; + if (!scan.hasTrailingBytes) + return true; + + std::error_code sizeError; + const std::uintmax_t fileSize = std::filesystem::file_size(path, sizeError); + if (sizeError) { + error = "unable to inspect interrupted dataset size: " + sizeError.message(); + return false; + } + + std::ifstream in(path, std::ios::binary); + if (!in) { + error = "unable to reopen interrupted dataset"; + return false; + } + + std::uintmax_t cursor = static_cast(scan.dataEnd); + in.seekg(scan.dataEnd); + if (!in) { + error = "unable to seek to interrupted dataset tail"; + return false; + } + + std::vector own; + std::vector partner; + std::uintmax_t validEnd = cursor; + + constexpr std::uintmax_t RECORD_METADATA_BYTES = 20; + + while (cursor < fileSize) { + const std::uintmax_t remaining = fileSize - cursor; + if (remaining < RECORD_METADATA_BYTES) + break; + + std::uint16_t ownCount = 0; + std::uint16_t partnerCount = 0; + float target = 0.0f; + std::int32_t teacherCp = 0; + std::uint32_t gameId = 0; + std::uint16_t ply = 0; + std::uint8_t side = 0; + std::uint8_t flags = 0; + + if (!readValue(in, ownCount) + || !readValue(in, partnerCount) + || !readValue(in, target) + || !readValue(in, teacherCp) + || !readValue(in, gameId) + || !readValue(in, ply) + || !readValue(in, side) + || !readValue(in, flags)) { + break; + } + + if (ownCount > scan.header.maxFeatures + || partnerCount > scan.header.maxFeatures + || side >= COLOR_NB + || !std::isfinite(target)) { + break; + } + + const std::uintmax_t featureBytes = + static_cast(ownCount + partnerCount) * sizeof(std::uint16_t); + if (remaining < RECORD_METADATA_BYTES + featureBytes) + break; + + if (!readFeatureVector(in, ownCount, own) + || !readFeatureVector(in, partnerCount, partner)) { + break; + } + + const bool validFeatures = std::all_of( + own.begin(), own.end(), + [&](std::uint16_t feature) { return feature < scan.header.featureCount; }) + && std::all_of( + partner.begin(), partner.end(), + [&](std::uint16_t feature) { return feature < scan.header.featureCount; }); + if (!validFeatures) + break; + + if (deduper != nullptr) + deduper->preload(featureFingerprint(own, partner, side)); + + if (!scan.hasGameId || gameId > scan.maxGameId) { + scan.maxGameId = gameId; + scan.hasGameId = true; + } + + ++recovered; + cursor += RECORD_METADATA_BYTES + featureBytes; + validEnd = cursor; + } + + in.close(); + + if (validEnd < fileSize) { + std::error_code resizeError; + std::filesystem::resize_file(path, validEnd, resizeError); + if (resizeError) { + error = "unable to trim incomplete interrupted record: " + resizeError.message(); + return false; + } + } + + if (recovered != 0) { + if (scan.header.recordCount > std::numeric_limits::max() - recovered) { + error = "dataset record count overflow during recovery"; + return false; + } + + scan.header.recordCount += recovered; + + std::fstream header(path, std::ios::binary | std::ios::in | std::ios::out); + if (!header) { + error = "unable to reopen dataset to commit recovered count"; + return false; + } + header.seekp(RECORD_COUNT_OFFSET); + if (!writeValue(header, scan.header.recordCount)) { + error = "unable to write recovered dataset count"; + return false; + } + header.flush(); + if (!header) { + error = "unable to flush recovered dataset count"; + return false; + } + } + + scan.dataEnd = static_cast(validEnd); + scan.hasTrailingBytes = false; + return true; +} + +TeacherResult searchPosition( + SearchEngine& search, + Position& pos, + const Options& options, + std::mt19937_64& rng, + int ply) { + + TeacherResult result; + if (stopRequested()) + return result; + + SearchLimits limits; + limits.depth = options.depth; + + bool explorationSearch = false; + if (options.exploration > 0.0 && ply <= EXPLORATION_MAX_PLY) { + std::uniform_real_distribution probability(0.0, 1.0); + explorationSearch = probability(rng) < options.exploration; + } + + const int requestedMultiPV = explorationSearch + ? std::clamp(options.explorationMultiPV, 2, SearchEngine::MAX_MULTI_PV) + : 1; + + std::ostringstream output; + if (requestedMultiPV > 1) + search.startMultiPV(pos, limits, requestedMultiPV, output); + else + search.start(pos, limits, output); + + while (search.searching()) { + if (stopRequested()) + search.stop(); + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + search.stopAndWait(); + + if (stopRequested()) + return result; + + int bestDepth = -1; + std::vector finalLines; + + std::istringstream lines(output.str()); + std::string line; + while (std::getline(lines, line)) { + int depth = -1; + RootCandidate candidate; + if (!parseInfoLine(pos, line, depth, candidate)) + continue; + + if (depth > bestDepth) { + bestDepth = depth; + finalLines.clear(); + } + if (depth == bestDepth) { + auto it = std::find_if( + finalLines.begin(), + finalLines.end(), + [&](const RootCandidate& item) { return item.rank == candidate.rank; }); + if (it == finalLines.end()) + finalLines.push_back(candidate); + else + *it = candidate; + } + } + + std::sort( + finalLines.begin(), + finalLines.end(), + [](const RootCandidate& a, const RootCandidate& b) { + return a.rank < b.rank; + }); + + if (finalLines.empty()) + return result; + + const RootCandidate& best = finalLines.front(); + result.ok = true; + result.bestMate = best.mate; + result.bestCp = best.cp; + result.move = best.move; + result.candidates = finalLines; + + if (best.mate || !explorationSearch || finalLines.size() < 2) + return result; + + std::vector alternatives; + std::vector weights; + + for (std::size_t i = 1; i < finalLines.size(); ++i) { + const RootCandidate& candidate = finalLines[i]; + if (candidate.mate) + continue; + + const int loss = best.cp - candidate.cp; + if (loss < 0 || loss > options.explorationMaxLoss) + continue; + + alternatives.push_back(&candidate); + weights.push_back(std::exp( + -static_cast(loss) / options.explorationTemperature)); + } + + if (alternatives.empty()) + return result; + + std::discrete_distribution choose(weights.begin(), weights.end()); + result.move = alternatives[choose(rng)]->move; + result.explored = true; + return result; +} + +void workerMain( + unsigned workerId, + const Options& options, + DatasetWriter& writer, + DatasetDeduper& deduper, + GenerationStats& stats, + std::atomic& nextGameId, + std::atomic& failed) { + + SearchEngine search; + if (!search.setHashSizeMB(options.hashMB)) { + std::cerr << "worker " << workerId << ": unable to allocate hash\n"; + failed.store(true, std::memory_order_relaxed); + return; + } + + std::mt19937_64 rng( + options.seed + + 0x9E3779B97F4A7C15ULL * (static_cast(workerId) + 1ULL)); + + Position pos; + Fen4State fenState; + std::string error; + const int firstSamplePly = std::max({1, options.randomPlies, options.skipPlies}); + + while (!stopRequested() + && !failed.load(std::memory_order_relaxed) + && writer.addedCount() < writer.targetAdded()) { + + const std::uint32_t gameId = nextGameId.fetch_add(1, std::memory_order_relaxed); + + if (!resetModern(pos, fenState, error)) { + std::cerr << "worker " << workerId << ": start position failed: " << error << '\n'; + failed.store(true, std::memory_order_relaxed); + return; + } + search.newGame(); + + for (int ply = 0; ply < options.maxPlies; ++ply) { + if (stopRequested() + || failed.load(std::memory_order_relaxed) + || writer.addedCount() >= writer.targetAdded()) + break; + + MoveList legal; + generateLegalMoves(pos, legal); + if (legal.size == 0) + break; + + Move move; + TeacherResult teacher; + + if (ply < options.randomPlies) { + move = chooseRandomOpeningMove(pos, rng); + } + else { + teacher = searchPosition(search, pos, options, rng, ply); + if (stopRequested()) + break; + if (!teacher.ok) + break; + + move = teacher.move; + if (teacher.explored) + stats.exploredMoves.fetch_add(1, std::memory_order_relaxed); + + const bool samplePly = ply >= firstSamplePly + && ((ply - firstSamplePly) % options.sampleEvery == 0); + + if (samplePly && !teacher.bestMate) { + const int clippedCp = std::clamp( + teacher.bestCp, + -options.cpClamp, + options.cpClamp); + + Record record; + record.target = static_cast(clippedCp) / options.scoreScale; + record.teacherCp = teacher.bestCp; + record.gameId = gameId; + record.ply = static_cast(std::min(ply, 65535)); + record.sideToMove = static_cast(pos.sideToMove()); + + if (pos.inCheck()) + record.flags |= 1u; + if (pos.aliveMask() != 0xF) + record.flags |= 2u; + if (teacher.explored) + record.flags |= 4u; + + const Color perspective = pos.sideToMove(); + const Color partner = static_cast( + static_cast(perspective) ^ 2u); + + collectFeatures(pos, perspective, pos.kingSquare(perspective), record.own); + collectFeatures(pos, perspective, pos.kingSquare(partner), record.partner); + + if (record.own.size() != record.partner.size()) { + std::cerr << "worker " << workerId + << ": feature stream size mismatch\n"; + failed.store(true, std::memory_order_relaxed); + return; + } + + bool unique = true; + if (options.dedup) { + unique = deduper.claim(featureFingerprint( + record.own, + record.partner, + record.sideToMove)); + } + + if (!unique) { + stats.duplicateSamples.fetch_add(1, std::memory_order_relaxed); + } + else if (!writer.tryWrite(record)) { + std::cerr << "worker " << workerId << ": dataset write failed\n"; + failed.store(true, std::memory_order_relaxed); + return; + } + else { + reportProgress(writer, stats, workerId); + } + } + } + + if (stopRequested() || !move) + break; + + const bool terminalKingCapture = isTerminalKingCapture(pos, move); + StateInfo state; + pos.makeMove(move, state); + + if (terminalKingCapture) + break; + } + } + + search.stopAndWait(); +} + +bool hasCommandLineOption(int argc, char** argv, std::string_view wanted) { + for (int i = 1; i < argc; ++i) + if (std::string_view(argv[i]) == wanted) + return true; + return false; +} + +std::vector filteredArgumentsWithoutResume(int argc, char** argv) { + std::vector args; + args.reserve(static_cast(argc)); + for (int i = 0; i < argc; ++i) { + if (i != 0 && std::string_view(argv[i]) == "--resume") + continue; + args.emplace_back(argv[i]); + } + return args; +} + +std::vector writableArgv(std::vector& args) { + std::vector result; + result.reserve(args.size()); + for (std::string& arg : args) + result.push_back(arg.data()); + return result; +} + +} // namespace +} // namespace ironphoenix + +int main(int argc, char** argv) { + using namespace ironphoenix; + + const bool resume = hasCommandLineOption(argc, argv, "--resume"); + const bool appendOnCommandLine = hasCommandLineOption(argc, argv, "--append"); + if (resume && appendOnCommandLine) { + std::cerr << "--resume and --append are different modes; choose only one\n"; + return 1; + } + + auto filteredArgs = filteredArgumentsWithoutResume(argc, argv); + auto filteredArgv = writableArgv(filteredArgs); + + Options options; + bool showedHelp = false; + if (!parseOptions( + static_cast(filteredArgv.size()), + filteredArgv.data(), + options, + showedHelp)) { + if (!showedHelp) + printUsage(); + return showedHelp ? 0 : 1; + } + + if (resume && showedHelp) + return 0; + + const std::uint64_t requestedPositionArgument = options.positions; + const std::filesystem::path outputPath(options.output); + const bool outputExists = std::filesystem::exists(outputPath); + + DatasetDeduper deduper; + DatasetScan existing; + DatasetScan* existingPtr = nullptr; + std::string error; + + const bool continueExisting = (options.append || resume) && outputExists; + if (continueExisting) { + if (!scanDataset(outputPath, existing, options.dedup ? &deduper : nullptr, error)) { + std::cerr << "cannot continue dataset: " << error << '\n'; + return 1; + } + + if (existing.hasTrailingBytes) { + std::uint64_t recovered = 0; + if (!recoverTrailingRecords( + outputPath, + existing, + options.dedup ? &deduper : nullptr, + recovered, + error)) { + std::cerr << "cannot recover interrupted dataset: " << error << '\n'; + return 1; + } + if (recovered != 0) + std::cout << "recovered " << recovered + << " complete positions from an interrupted run\n"; + else + std::cout << "trimmed incomplete interrupted dataset tail\n"; + } + + if (existing.header.depth != static_cast(options.depth)) { + std::cerr << "cannot continue: teacher depth differs (existing " + << existing.header.depth << ", requested " << options.depth << ")\n"; + return 1; + } + if (std::fabs(existing.header.scoreScale - options.scoreScale) > 1e-6f) { + std::cerr << "cannot continue: score scale differs\n"; + return 1; + } + + if (!options.seedExplicit) { + options.seed = existing.header.seed + ^ mix64(existing.header.recordCount + 1ULL) + ^ 0xA5A5A5A55A5A5A5AULL; + } + + existingPtr = &existing; + + if (resume) { + if (existing.header.recordCount >= requestedPositionArgument) { + std::cout << "resume target already reached: dataset has " + << existing.header.recordCount << " positions (target " + << requestedPositionArgument << ")\n"; + return 0; + } + + options.positions = requestedPositionArgument - existing.header.recordCount; + std::cout << "resume mode: existing " << existing.header.recordCount + << ", target total " << requestedPositionArgument + << ", remaining " << options.positions << '\n'; + } + else { + std::cout << "append mode: existing positions " + << existing.header.recordCount + << ", adding " << options.positions << '\n'; + } + + if (options.dedup) + std::cout << "preloaded dedup fingerprints " << deduper.size() << '\n'; + } + else if (resume) { + std::cout << "resume requested but dataset does not exist; creating it toward total target " + << requestedPositionArgument << '\n'; + } + else if (options.append) { + std::cout << "append requested but dataset does not exist; creating it\n"; + } + + const std::uint64_t intendedTotal = resume + ? requestedPositionArgument + : (existingPtr != nullptr ? existing.header.recordCount : 0ULL) + options.positions; + + DatasetWriter writer(options, existingPtr, error); + if (!writer.good()) { + std::cerr << error << '\n'; + return 1; + } + + gStopRequested = 0; + gLastReported.store(0, std::memory_order_relaxed); + std::signal(SIGINT, handleInterrupt); +#ifdef SIGTERM + std::signal(SIGTERM, handleInterrupt); +#endif + + std::cout + << "PhoenixNet dataset generation\n" + << "output " << options.output << '\n' + << "mode " << (resume ? "resume" : (options.append ? "append" : "new")) << '\n' + << "new positions " << options.positions << '\n'; + if (resume) + std::cout << "target total " << requestedPositionArgument << '\n'; + std::cout + << "depth " << options.depth << '\n' + << "workers " << options.workers << '\n' + << "hash/worker " << options.hashMB << " MB\n" + << "random plies " << options.randomPlies << '\n' + << "first sample " << std::max({1, options.randomPlies, options.skipPlies}) << '\n' + << "sample every " << options.sampleEvery << '\n' + << "exploration " << options.exploration << '\n' + << "explore MultiPV " << options.explorationMultiPV << '\n' + << "explore max ply " << EXPLORATION_MAX_PLY << '\n' + << "explore maxloss " << options.explorationMaxLoss << " cp\n" + << "dedup " << (options.dedup ? "on" : "off") << '\n' + << "progress every 10 positions\n" + << "seed " << options.seed << "\n\n" + << "Press Ctrl+C once to stop safely and commit the current dataset.\n\n"; + + std::atomic nextGameId{0}; + std::atomic failed{false}; + GenerationStats stats; + std::vector workers; + workers.reserve(options.workers); + + for (unsigned workerId = 0; workerId < options.workers; ++workerId) { + workers.emplace_back( + workerMain, + workerId, + std::cref(options), + std::ref(writer), + std::ref(deduper), + std::ref(stats), + std::ref(nextGameId), + std::ref(failed)); + } + + for (auto& worker : workers) + worker.join(); + + std::signal(SIGINT, SIG_DFL); +#ifdef SIGTERM + std::signal(SIGTERM, SIG_DFL); +#endif + + if (!writer.commit(error)) { + std::cerr << "dataset commit failed: " << error << '\n'; + return 1; + } + + if (failed.load(std::memory_order_relaxed)) { + std::cerr << "dataset generation encountered an error, but " + << writer.totalCount() << " completed positions were committed safely\n"; + return 1; + } + + if (stopRequested()) { + std::cout + << "stopped safely: added " << writer.addedCount() + << " positions this run; total " << writer.totalCount() + << ". Resume with --resume --positions " << intendedTotal + << " using the same output file.\n"; + return 0; + } + + std::cout + << "done: added " << writer.addedCount() + << " unique positions; total " << writer.totalCount() + << "; skipped duplicates " << stats.duplicateSamples.load(std::memory_order_relaxed) + << "; explored moves " << stats.exploredMoves.load(std::memory_order_relaxed) + << '\n'; + + return 0; +} diff --git a/tools/nnue_dataset_with_append.cpp b/tools/nnue_dataset_with_append.cpp new file mode 100644 index 0000000..0918863 --- /dev/null +++ b/tools/nnue_dataset_with_append.cpp @@ -0,0 +1,1207 @@ +#include "ironphoenix/fen.hpp" +#include "ironphoenix/movegen.hpp" +#include "ironphoenix/nnue.hpp" +#include "ironphoenix/search.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ironphoenix { +namespace { + +constexpr std::array DATA_MAGIC = {'I', 'P', 'D', 'A', 'T', 'A', '1', '\0'}; +constexpr std::uint32_t DATA_VERSION = 1; +constexpr std::uint32_t MAX_FEATURES_PER_STREAM = 64; +constexpr std::streamoff RECORD_COUNT_OFFSET = 36; + +constexpr std::string_view MODERN_START_FEN = + "R-0,0,0,0-1,1,1,1-1,1,1,1-0,0,0,0-0-" + "x,x,x,yR,yN,yB,yK,yQ,yB,yN,yR,x,x,x/" + "x,x,x,yP,yP,yP,yP,yP,yP,yP,yP,x,x,x/" + "x,x,x,8,x,x,x/" + "bR,bP,10,gP,gR/" + "bN,bP,10,gP,gN/" + "bB,bP,10,gP,gB/" + "bQ,bP,10,gP,gK/" + "bK,bP,10,gP,gQ/" + "bB,bP,10,gP,gB/" + "bN,bP,10,gP,gN/" + "bR,bP,10,gP,gR/" + "x,x,x,8,x,x,x/" + "x,x,x,rP,rP,rP,rP,rP,rP,rP,rP,x,x,x/" + "x,x,x,rR,rN,rB,rQ,rK,rB,rN,rR,x,x,x"; + +struct Options final { + std::string output = "phoenix_dataset.ipd"; + std::uint64_t positions = 500000; + int depth = 8; + int randomPlies = 8; + int skipPlies = 12; + int sampleEvery = 3; + int maxPlies = 320; + int cpClamp = 4000; + float scoreScale = 400.0f; + std::size_t hashMB = 16; + unsigned workers = 1; + std::uint64_t seed = 0x49524f4e50484f45ULL; + bool seedExplicit = false; + bool append = false; + bool dedup = true; + + // Controlled exploration: normally play MultiPV #1. With this probability, + // sample a nearby alternative from the top MultiPV lines. + double exploration = 0.15; + int explorationMultiPV = 4; + double explorationTemperature = 120.0; + int explorationMaxLoss = 250; +}; + +struct DatasetHeader final { + std::array magic{}; + std::uint32_t version = 0; + std::uint32_t featureCount = 0; + std::uint32_t maxFeatures = 0; + std::uint32_t depth = 0; + float scoreScale = 0.0f; + std::uint64_t seed = 0; + std::uint64_t recordCount = 0; + std::streamoff dataOffset = 0; +}; + +struct DatasetScan final { + DatasetHeader header{}; + std::uint32_t maxGameId = 0; + bool hasGameId = false; + std::streamoff dataEnd = 0; + bool hasTrailingBytes = false; +}; + +struct Record final { + std::vector own; + std::vector partner; + float target = 0.0f; + std::int32_t teacherCp = 0; + std::uint32_t gameId = 0; + std::uint16_t ply = 0; + std::uint8_t sideToMove = 0; + std::uint8_t flags = 0; +}; + +struct RootCandidate final { + Move move{}; + int cp = 0; + bool mate = false; + int rank = 1; +}; + +struct TeacherResult final { + bool ok = false; + bool bestMate = false; + int bestCp = 0; + Move move{}; + bool explored = false; + std::vector candidates; +}; + +struct GenerationStats final { + std::atomic duplicateSamples{0}; + std::atomic exploredMoves{0}; +}; + +template +bool readValue(std::istream& in, T& value) { + return static_cast(in.read(reinterpret_cast(&value), sizeof(T))); +} + +template +bool writeValue(std::ostream& out, const T& value) { + out.write(reinterpret_cast(&value), sizeof(T)); + return static_cast(out); +} + +std::uint64_t mix64(std::uint64_t x) noexcept { + x += 0x9E3779B97F4A7C15ULL; + x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ULL; + x = (x ^ (x >> 27)) * 0x94D049BB133111EBULL; + return x ^ (x >> 31); +} + +std::uint64_t featureFingerprint( + const std::vector& own, + const std::vector& partner, + std::uint8_t sideToMove) noexcept { + + std::uint64_t hash = mix64(static_cast(sideToMove) + 0x50484f454e4958ULL); + hash ^= mix64(static_cast(own.size()) << 32); + + for (std::uint16_t feature : own) + hash = mix64(hash ^ (static_cast(feature) + 0x10001ULL)); + + hash = mix64(hash ^ 0xA55AA55AA55AA55AULL); + hash ^= mix64(static_cast(partner.size()) << 33); + + for (std::uint16_t feature : partner) + hash = mix64(hash ^ (static_cast(feature) + 0x20003ULL)); + + return hash; +} + +class DatasetDeduper final { +public: + void reserve(std::size_t count) { + std::lock_guard lock(mutex_); + seen_.reserve(count); + } + + void preload(std::uint64_t fingerprint) { + seen_.insert(fingerprint); + } + + bool claim(std::uint64_t fingerprint) { + std::lock_guard lock(mutex_); + return seen_.insert(fingerprint).second; + } + + [[nodiscard]] std::size_t size() const { + std::lock_guard lock(mutex_); + return seen_.size(); + } + +private: + mutable std::mutex mutex_; + std::unordered_set seen_; +}; + +bool readHeader(std::istream& in, DatasetHeader& header, std::string& error) { + if (!in.read(header.magic.data(), static_cast(header.magic.size()))) { + error = "dataset header is truncated"; + return false; + } + if (header.magic != DATA_MAGIC) { + error = "not an IronPhoenix IPDATA1 dataset"; + return false; + } + + if (!readValue(in, header.version) + || !readValue(in, header.featureCount) + || !readValue(in, header.maxFeatures) + || !readValue(in, header.depth) + || !readValue(in, header.scoreScale) + || !readValue(in, header.seed) + || !readValue(in, header.recordCount)) { + error = "dataset header is truncated"; + return false; + } + + if (header.version != DATA_VERSION) { + error = "unsupported dataset version " + std::to_string(header.version); + return false; + } + if (header.featureCount != NNUE::FEATURE_COUNT) { + error = "dataset feature count does not match this PhoenixNet build"; + return false; + } + if (header.maxFeatures != MAX_FEATURES_PER_STREAM) { + error = "dataset maximum sparse feature count is incompatible"; + return false; + } + if (!std::isfinite(header.scoreScale) || header.scoreScale <= 0.0f) { + error = "dataset score scale is invalid"; + return false; + } + + header.dataOffset = static_cast(in.tellg()); + return true; +} + +bool readFeatureVector( + std::istream& in, + std::uint16_t count, + std::vector& output) { + + output.resize(count); + if (count == 0) + return true; + + return static_cast(in.read( + reinterpret_cast(output.data()), + static_cast(count * sizeof(std::uint16_t)))); +} + +bool scanDataset( + const std::filesystem::path& path, + DatasetScan& scan, + DatasetDeduper* deduper, + std::string& error) { + + std::ifstream in(path, std::ios::binary); + if (!in) { + error = "unable to open dataset"; + return false; + } + + if (!readHeader(in, scan.header, error)) + return false; + + if (deduper != nullptr) + deduper->reserve(static_cast(std::min( + scan.header.recordCount * 2ULL + 1024ULL, + static_cast(std::numeric_limits::max())))); + + std::vector own; + std::vector partner; + + for (std::uint64_t i = 0; i < scan.header.recordCount; ++i) { + std::uint16_t ownCount = 0; + std::uint16_t partnerCount = 0; + float target = 0.0f; + std::int32_t teacherCp = 0; + std::uint32_t gameId = 0; + std::uint16_t ply = 0; + std::uint8_t side = 0; + std::uint8_t flags = 0; + + if (!readValue(in, ownCount) + || !readValue(in, partnerCount) + || !readValue(in, target) + || !readValue(in, teacherCp) + || !readValue(in, gameId) + || !readValue(in, ply) + || !readValue(in, side) + || !readValue(in, flags)) { + error = "dataset is truncated while reading record " + std::to_string(i); + return false; + } + + if (ownCount > scan.header.maxFeatures || partnerCount > scan.header.maxFeatures) { + error = "dataset record has an invalid sparse feature count"; + return false; + } + + if (!readFeatureVector(in, ownCount, own) + || !readFeatureVector(in, partnerCount, partner)) { + error = "dataset is truncated inside sparse features"; + return false; + } + + if (deduper != nullptr) + deduper->preload(featureFingerprint(own, partner, side)); + + if (!scan.hasGameId || gameId > scan.maxGameId) { + scan.maxGameId = gameId; + scan.hasGameId = true; + } + } + + scan.dataEnd = static_cast(in.tellg()); + char extra = 0; + scan.hasTrailingBytes = static_cast(in.read(&extra, 1)); + return true; +} + +class DatasetWriter final { +public: + DatasetWriter( + const Options& options, + const DatasetScan* existing, + std::string& error) + : targetAdded_(options.positions) { + + const std::filesystem::path path(options.output); + + if (existing != nullptr) { + if (existing->hasGameId + && existing->maxGameId == std::numeric_limits::max()) { + error = "cannot append: game id space is exhausted"; + return; + } + + baseCount_ = existing->header.recordCount; + gameOffset_ = existing->hasGameId ? existing->maxGameId + 1u : 0u; + countPosition_ = RECORD_COUNT_OFFSET; + + stream_.open(path, std::ios::binary | std::ios::in | std::ios::out); + if (!stream_) { + error = "unable to open existing dataset for append"; + return; + } + stream_.seekp(existing->dataEnd); + if (!stream_) { + error = "unable to seek to dataset append position"; + return; + } + } + else { + stream_.open( + path, + std::ios::binary | std::ios::in | std::ios::out | std::ios::trunc); + if (!stream_) { + error = "unable to create output dataset"; + return; + } + + stream_.write(DATA_MAGIC.data(), static_cast(DATA_MAGIC.size())); + writeValue(stream_, DATA_VERSION); + writeValue(stream_, static_cast(NNUE::FEATURE_COUNT)); + writeValue(stream_, MAX_FEATURES_PER_STREAM); + writeValue(stream_, static_cast(options.depth)); + writeValue(stream_, options.scoreScale); + writeValue(stream_, options.seed); + countPosition_ = static_cast(stream_.tellp()); + const std::uint64_t zero = 0; + writeValue(stream_, zero); + + if (!stream_) { + error = "failed while writing dataset header"; + return; + } + } + + good_ = true; + } + + ~DatasetWriter() { + if (stream_) + stream_.flush(); + } + + [[nodiscard]] bool good() const noexcept { return good_; } + [[nodiscard]] std::uint64_t addedCount() const noexcept { + return added_.load(std::memory_order_relaxed); + } + [[nodiscard]] std::uint64_t totalCount() const noexcept { + return baseCount_ + addedCount(); + } + [[nodiscard]] std::uint64_t targetAdded() const noexcept { return targetAdded_; } + + bool tryWrite(const Record& record) { + if (record.own.size() > MAX_FEATURES_PER_STREAM + || record.partner.size() > MAX_FEATURES_PER_STREAM) + return false; + + std::uint64_t current = added_.load(std::memory_order_relaxed); + while (current < targetAdded_) { + if (added_.compare_exchange_weak( + current, + current + 1, + std::memory_order_relaxed, + std::memory_order_relaxed)) + break; + } + + if (current >= targetAdded_) + return true; + + const std::uint64_t adjustedGameId64 = + static_cast(gameOffset_) + record.gameId; + if (adjustedGameId64 > std::numeric_limits::max()) + return false; + const auto adjustedGameId = static_cast(adjustedGameId64); + + std::lock_guard lock(mutex_); + if (!stream_ || committed_) + return false; + + const auto ownCount = static_cast(record.own.size()); + const auto partnerCount = static_cast(record.partner.size()); + + writeValue(stream_, ownCount); + writeValue(stream_, partnerCount); + writeValue(stream_, record.target); + writeValue(stream_, record.teacherCp); + writeValue(stream_, adjustedGameId); + writeValue(stream_, record.ply); + writeValue(stream_, record.sideToMove); + writeValue(stream_, record.flags); + + if (!record.own.empty()) { + stream_.write( + reinterpret_cast(record.own.data()), + static_cast(record.own.size() * sizeof(std::uint16_t))); + } + if (!record.partner.empty()) { + stream_.write( + reinterpret_cast(record.partner.data()), + static_cast(record.partner.size() * sizeof(std::uint16_t))); + } + + return static_cast(stream_); + } + + bool commit(std::string& error) { + std::lock_guard lock(mutex_); + if (committed_) + return true; + if (!stream_) { + error = "dataset stream is not writable"; + return false; + } + + // Flush record bytes first. The authoritative count is updated last. + // If the process dies before the count update, a future append trims + // the uncommitted trailing bytes discovered by scanDataset(). + stream_.flush(); + if (!stream_) { + error = "failed to flush dataset records"; + return false; + } + + const std::uint64_t count = totalCount(); + stream_.seekp(countPosition_); + if (!writeValue(stream_, count)) { + error = "failed to update dataset record count"; + return false; + } + stream_.flush(); + if (!stream_) { + error = "failed to commit dataset record count"; + return false; + } + + committed_ = true; + return true; + } + +private: + std::fstream stream_; + std::streamoff countPosition_ = RECORD_COUNT_OFFSET; + std::uint64_t baseCount_ = 0; + std::uint64_t targetAdded_ = 0; + std::uint32_t gameOffset_ = 0; + std::atomic added_{0}; + std::mutex mutex_; + bool good_ = false; + bool committed_ = false; +}; + +Direction kingSideDirection(Color c) noexcept { + switch (c) { + case RED: return EAST; + case BLUE: return SOUTH; + case YELLOW: return WEST; + case GREEN: return NORTH; + } + return EAST; +} + +Square findCastlingRook(const Position& pos, Color c, Square king, Direction direction) noexcept { + for (Square sq = Geometry::step(king, direction); + sq != SQ_NONE; + sq = Geometry::step(sq, direction)) { + if (pos.pieceAt(sq) == makePiece(c, ROOK)) + return sq; + } + return SQ_NONE; +} + +void configureStandardCastling(Position& pos) noexcept { + for (unsigned ci = 0; ci < COLOR_NB; ++ci) { + const Color c = static_cast(ci); + const Square king = pos.kingSquare(c); + if (king == SQ_NONE) + continue; + + for (unsigned laneIndex = 0; laneIndex < 2; ++laneIndex) { + Direction direction = kingSideDirection(c); + if (laneIndex == 1) + direction = Geometry::opposite(direction); + + const Square rook = findCastlingRook(pos, c, king, direction); + const Square rookTo = Geometry::step(king, direction); + const Square kingTo = rookTo == SQ_NONE + ? SQ_NONE + : Geometry::step(rookTo, direction); + + CastleLane lane{}; + if (rook != SQ_NONE && rookTo != SQ_NONE && kingTo != SQ_NONE && kingTo != rook) { + lane.kingFrom = king; + lane.rookFrom = rook; + lane.kingTo = kingTo; + lane.rookTo = rookTo; + lane.rightBit = static_cast(2u * ci + laneIndex); + } + pos.configureCastle(c, laneIndex, lane); + } + } +} + +bool resetModern(Position& pos, Fen4State& state, std::string& error) { + pos.setRulesetId(1); + if (!setFromFen4(pos, MODERN_START_FEN, state, error)) + return false; + configureStandardCastling(pos); + return true; +} + +void collectFeatures( + const Position& pos, + Color perspective, + Square anchorKing, + std::vector& output) { + + output.clear(); + output.reserve(MAX_FEATURES_PER_STREAM); + + for (unsigned ci = 0; ci < COLOR_NB; ++ci) { + const Color pieceColor = static_cast(ci); + for (unsigned pti = PAWN; pti <= KING; ++pti) { + const PieceType pt = static_cast(pti); + Bitboard pieces = pos.pieces(pieceColor, pt); + + while (pieces) { + const Square sq = popLsb(pieces); + const std::size_t feature = NNUE::featureIndex( + perspective, + anchorKing, + makePiece(pieceColor, pt), + sq); + + if (feature < NNUE::FEATURE_COUNT + && feature <= std::numeric_limits::max()) { + output.push_back(static_cast(feature)); + } + } + } + } +} + +bool parseInfoLine( + Position& pos, + const std::string& line, + int& depth, + RootCandidate& candidate) { + + if (line.rfind("info ", 0) != 0) + return false; + + std::istringstream input(line); + std::string token; + input >> token; // info + + int parsedDepth = -1; + int rank = 1; + bool haveScore = false; + bool mate = false; + int score = 0; + std::string pvMove; + + while (input >> token) { + if (token == "depth") { + input >> parsedDepth; + } + else if (token == "multipv") { + input >> rank; + } + else if (token == "score") { + std::string kind; + input >> kind >> score; + if (!input) + return false; + mate = kind == "mate"; + haveScore = kind == "cp" || kind == "mate"; + } + else if (token == "pv") { + input >> pvMove; + break; + } + } + + if (parsedDepth < 0 || rank < 1 || !haveScore || pvMove.empty()) + return false; + + Move move; + std::string error; + if (!parseLegalMove(pos, pvMove, move, error)) + return false; + + depth = parsedDepth; + candidate.move = move; + candidate.rank = rank; + candidate.mate = mate; + candidate.cp = mate + ? (score > 0 ? 30000 - std::abs(score) : -30000 + std::abs(score)) + : score; + return true; +} + +TeacherResult searchPosition( + SearchEngine& search, + Position& pos, + const Options& options, + std::mt19937_64& rng) { + + SearchLimits limits; + limits.depth = options.depth; + + const int requestedMultiPV = options.exploration > 0.0 + ? std::clamp(options.explorationMultiPV, 2, SearchEngine::MAX_MULTI_PV) + : 1; + + std::ostringstream output; + if (requestedMultiPV > 1) + search.startMultiPV(pos, limits, requestedMultiPV, output); + else + search.start(pos, limits, output); + + while (search.searching()) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + search.stopAndWait(); + + int bestDepth = -1; + std::vector finalLines; + std::string bestMoveText; + + std::istringstream lines(output.str()); + std::string line; + while (std::getline(lines, line)) { + if (line.rfind("bestmove ", 0) == 0) { + bestMoveText = line.substr(9); + continue; + } + + int depth = -1; + RootCandidate candidate; + if (!parseInfoLine(pos, line, depth, candidate)) + continue; + + if (depth > bestDepth) { + bestDepth = depth; + finalLines.clear(); + } + if (depth == bestDepth) { + auto it = std::find_if( + finalLines.begin(), + finalLines.end(), + [&](const RootCandidate& item) { return item.rank == candidate.rank; }); + if (it == finalLines.end()) + finalLines.push_back(candidate); + else + *it = candidate; + } + } + + std::sort( + finalLines.begin(), + finalLines.end(), + [](const RootCandidate& a, const RootCandidate& b) { + return a.rank < b.rank; + }); + + TeacherResult result; + if (finalLines.empty()) + return result; + + const RootCandidate& best = finalLines.front(); + result.ok = true; + result.bestMate = best.mate; + result.bestCp = best.cp; + result.move = best.move; + result.candidates = finalLines; + + if (best.mate || options.exploration <= 0.0 || finalLines.size() < 2) + return result; + + std::uniform_real_distribution probability(0.0, 1.0); + if (probability(rng) >= options.exploration) + return result; + + std::vector alternatives; + std::vector weights; + + for (std::size_t i = 1; i < finalLines.size(); ++i) { + const RootCandidate& candidate = finalLines[i]; + if (candidate.mate) + continue; + + const int loss = best.cp - candidate.cp; + if (loss < 0 || loss > options.explorationMaxLoss) + continue; + + const double exponent = -static_cast(loss) / options.explorationTemperature; + alternatives.push_back(&candidate); + weights.push_back(std::exp(exponent)); + } + + if (alternatives.empty()) + return result; + + std::discrete_distribution choose(weights.begin(), weights.end()); + result.move = alternatives[choose(rng)]->move; + result.explored = true; + return result; +} + +Move chooseRandomOpeningMove(Position& pos, std::mt19937_64& rng) { + MoveList legal; + generateLegalMoves(pos, legal); + if (legal.size == 0) + return {}; + + std::array candidates{}; + std::size_t candidateCount = 0; + for (Move move : legal) { + if (!isTerminalKingCapture(pos, move)) + candidates[candidateCount++] = move; + } + + if (candidateCount == 0) + return legal.moves[0]; + + std::uniform_int_distribution pick(0, candidateCount - 1); + return candidates[pick(rng)]; +} + +void workerMain( + unsigned workerId, + const Options& options, + DatasetWriter& writer, + DatasetDeduper& deduper, + GenerationStats& stats, + std::atomic& nextGameId, + std::atomic& failed) { + + SearchEngine search; + if (!search.setHashSizeMB(options.hashMB)) { + std::cerr << "worker " << workerId << ": unable to allocate hash\n"; + failed.store(true, std::memory_order_relaxed); + return; + } + + std::mt19937_64 rng( + options.seed + + 0x9E3779B97F4A7C15ULL * (static_cast(workerId) + 1ULL)); + + Position pos; + Fen4State fenState; + std::string error; + + const int firstSamplePly = std::max({1, options.randomPlies, options.skipPlies}); + + while (!failed.load(std::memory_order_relaxed) + && writer.addedCount() < writer.targetAdded()) { + + const std::uint32_t gameId = nextGameId.fetch_add(1, std::memory_order_relaxed); + + if (!resetModern(pos, fenState, error)) { + std::cerr << "worker " << workerId << ": start position failed: " << error << '\n'; + failed.store(true, std::memory_order_relaxed); + return; + } + search.newGame(); + + for (int ply = 0; ply < options.maxPlies; ++ply) { + if (failed.load(std::memory_order_relaxed) + || writer.addedCount() >= writer.targetAdded()) + break; + + MoveList legal; + generateLegalMoves(pos, legal); + if (legal.size == 0) + break; + + Move move; + TeacherResult teacher; + + if (ply < options.randomPlies) { + move = chooseRandomOpeningMove(pos, rng); + } + else { + teacher = searchPosition(search, pos, options, rng); + if (!teacher.ok) + break; + + move = teacher.move; + if (teacher.explored) + stats.exploredMoves.fetch_add(1, std::memory_order_relaxed); + + const bool samplePly = ply >= firstSamplePly + && ((ply - firstSamplePly) % options.sampleEvery == 0); + + if (samplePly && !teacher.bestMate) { + const int clippedCp = std::clamp( + teacher.bestCp, + -options.cpClamp, + options.cpClamp); + + Record record; + record.target = static_cast(clippedCp) / options.scoreScale; + record.teacherCp = teacher.bestCp; + record.gameId = gameId; + record.ply = static_cast(std::min(ply, 65535)); + record.sideToMove = static_cast(pos.sideToMove()); + + if (pos.inCheck()) + record.flags |= 1u; + if (pos.aliveMask() != 0xF) + record.flags |= 2u; + if (teacher.explored) + record.flags |= 4u; + + const Color perspective = pos.sideToMove(); + const Color partner = static_cast( + static_cast(perspective) ^ 2u); + + collectFeatures( + pos, + perspective, + pos.kingSquare(perspective), + record.own); + collectFeatures( + pos, + perspective, + pos.kingSquare(partner), + record.partner); + + if (record.own.size() != record.partner.size()) { + std::cerr << "worker " << workerId + << ": feature stream size mismatch\n"; + failed.store(true, std::memory_order_relaxed); + return; + } + + bool unique = true; + if (options.dedup) { + const std::uint64_t fingerprint = featureFingerprint( + record.own, + record.partner, + record.sideToMove); + unique = deduper.claim(fingerprint); + } + + if (!unique) { + stats.duplicateSamples.fetch_add(1, std::memory_order_relaxed); + } + else if (!writer.tryWrite(record)) { + std::cerr << "worker " << workerId << ": dataset write failed\n"; + failed.store(true, std::memory_order_relaxed); + return; + } + else { + const auto count = writer.addedCount(); + if (count != 0 && (count % 1000 == 0 || count == writer.targetAdded())) { + std::cout << "new positions " << count << '/' << writer.targetAdded() + << " total " << writer.totalCount() + << " duplicates " << stats.duplicateSamples.load(std::memory_order_relaxed) + << " explored " << stats.exploredMoves.load(std::memory_order_relaxed) + << " worker " << workerId << '\n'; + } + } + } + } + + if (!move) + break; + + const bool terminalKingCapture = isTerminalKingCapture(pos, move); + StateInfo state; + pos.makeMove(move, state); + + if (terminalKingCapture) + break; + } + } +} + +bool parseUnsigned(std::string_view text, std::uint64_t& value) { + try { + std::size_t used = 0; + value = std::stoull(std::string(text), &used); + return used == text.size(); + } + catch (...) { + return false; + } +} + +bool parseSigned(std::string_view text, int& value) { + try { + std::size_t used = 0; + value = std::stoi(std::string(text), &used); + return used == text.size(); + } + catch (...) { + return false; + } +} + +bool parseDouble(std::string_view text, double& value) { + try { + std::size_t used = 0; + value = std::stod(std::string(text), &used); + return used == text.size() && std::isfinite(value); + } + catch (...) { + return false; + } +} + +void printUsage() { + std::cout + << "IronPhoenix PhoenixNet dataset generator\n\n" + << "Usage: ironphoenix_dataset [options]\n\n" + << " --output Output .ipd file\n" + << " --positions New samples to generate (default 500000)\n" + << " --append Safely append to an existing compatible file\n" + << " --depth Teacher search depth (default 8)\n" + << " --workers Parallel self-play workers (default 1)\n" + << " --hash Hash MB per worker (default 16)\n" + << " --random-plies Random opening plies (default 8)\n" + << " --skip-plies Earliest sample ply (default 12)\n" + << " --sample-every Save one of every N plies (default 3)\n" + << " --max-plies Maximum game length (default 320)\n" + << " --cp-clamp Teacher CP clamp (default 4000)\n" + << " --score-scale CP per target unit (default 400)\n" + << " --exploration <0..1> Chance to choose a strong non-best move (default 0.15)\n" + << " --exploration-multipv Root lines considered for exploration (default 4)\n" + << " --exploration-temperature N Softmax temperature in CP (default 120)\n" + << " --exploration-max-loss Largest alternative loss allowed (default 250)\n" + << " --dedup Enable feature-position deduplication (default)\n" + << " --no-dedup Disable deduplication\n" + << " --seed RNG seed\n" + << " --help Show this help\n"; +} + +bool parseOptions(int argc, char** argv, Options& options, bool& showedHelp) { + showedHelp = false; + + for (int i = 1; i < argc; ++i) { + const std::string_view arg = argv[i]; + auto value = [&]() -> std::string_view { + if (i + 1 >= argc) + return {}; + return argv[++i]; + }; + + if (arg == "--help" || arg == "-h") { + printUsage(); + showedHelp = true; + return false; + } + if (arg == "--append") { + options.append = true; + } + else if (arg == "--dedup") { + options.dedup = true; + } + else if (arg == "--no-dedup") { + options.dedup = false; + } + else if (arg == "--output") { + const auto text = value(); + if (text.empty()) return false; + options.output = text; + } + else if (arg == "--positions") { + std::uint64_t parsed = 0; + if (!parseUnsigned(value(), parsed) || parsed == 0) return false; + options.positions = parsed; + } + else if (arg == "--depth") { + if (!parseSigned(value(), options.depth) + || options.depth < 1 || options.depth > 32) return false; + } + else if (arg == "--workers") { + std::uint64_t parsed = 0; + if (!parseUnsigned(value(), parsed) || parsed == 0 || parsed > 128) return false; + options.workers = static_cast(parsed); + } + else if (arg == "--hash") { + std::uint64_t parsed = 0; + if (!parseUnsigned(value(), parsed) || parsed == 0 || parsed > 4096) return false; + options.hashMB = static_cast(parsed); + } + else if (arg == "--random-plies") { + if (!parseSigned(value(), options.randomPlies) || options.randomPlies < 0) return false; + } + else if (arg == "--skip-plies") { + if (!parseSigned(value(), options.skipPlies) || options.skipPlies < 0) return false; + } + else if (arg == "--sample-every") { + if (!parseSigned(value(), options.sampleEvery) || options.sampleEvery < 1) return false; + } + else if (arg == "--max-plies") { + if (!parseSigned(value(), options.maxPlies) || options.maxPlies < 1) return false; + } + else if (arg == "--cp-clamp") { + if (!parseSigned(value(), options.cpClamp) + || options.cpClamp < 1 || options.cpClamp >= 29000) return false; + } + else if (arg == "--score-scale") { + double parsed = 0.0; + if (!parseDouble(value(), parsed) || parsed <= 0.0) return false; + options.scoreScale = static_cast(parsed); + } + else if (arg == "--exploration") { + if (!parseDouble(value(), options.exploration) + || options.exploration < 0.0 || options.exploration > 1.0) return false; + } + else if (arg == "--exploration-multipv") { + if (!parseSigned(value(), options.explorationMultiPV) + || options.explorationMultiPV < 2 + || options.explorationMultiPV > SearchEngine::MAX_MULTI_PV) return false; + } + else if (arg == "--exploration-temperature") { + if (!parseDouble(value(), options.explorationTemperature) + || options.explorationTemperature <= 0.0) return false; + } + else if (arg == "--exploration-max-loss") { + if (!parseSigned(value(), options.explorationMaxLoss) + || options.explorationMaxLoss < 0) return false; + } + else if (arg == "--seed") { + if (!parseUnsigned(value(), options.seed)) return false; + options.seedExplicit = true; + } + else { + std::cerr << "unknown option: " << arg << '\n'; + return false; + } + } + + return true; +} + +} // namespace +} // namespace ironphoenix + +int main(int argc, char** argv) { + using namespace ironphoenix; + + Options options; + bool showedHelp = false; + if (!parseOptions(argc, argv, options, showedHelp)) { + if (!showedHelp) + printUsage(); + return showedHelp ? 0 : 1; + } + + const std::filesystem::path outputPath(options.output); + DatasetDeduper deduper; + DatasetScan existing; + DatasetScan* existingPtr = nullptr; + std::string error; + + if (options.append && std::filesystem::exists(outputPath)) { + if (!scanDataset(outputPath, existing, options.dedup ? &deduper : nullptr, error)) { + std::cerr << "cannot append: " << error << '\n'; + return 1; + } + + if (existing.header.depth != static_cast(options.depth)) { + std::cerr << "cannot append: teacher depth differs (existing " + << existing.header.depth << ", requested " << options.depth << ")\n"; + return 1; + } + if (std::fabs(existing.header.scoreScale - options.scoreScale) > 1e-6f) { + std::cerr << "cannot append: score scale differs\n"; + return 1; + } + + if (existing.hasTrailingBytes) { + std::error_code resizeError; + std::filesystem::resize_file( + outputPath, + static_cast(existing.dataEnd), + resizeError); + if (resizeError) { + std::cerr << "cannot trim stale uncommitted bytes: " + << resizeError.message() << '\n'; + return 1; + } + std::cout << "trimmed stale uncommitted bytes from interrupted append\n"; + } + + if (!options.seedExplicit) { + options.seed = existing.header.seed + ^ mix64(existing.header.recordCount + 1ULL) + ^ 0xA5A5A5A55A5A5A5AULL; + } + + existingPtr = &existing; + std::cout << "append mode: existing positions " + << existing.header.recordCount << '\n'; + if (options.dedup) + std::cout << "preloaded dedup fingerprints " << deduper.size() << '\n'; + } + else if (options.append) { + std::cout << "append requested but dataset does not exist; creating it\n"; + } + + DatasetWriter writer(options, existingPtr, error); + if (!writer.good()) { + std::cerr << error << '\n'; + return 1; + } + + std::cout + << "PhoenixNet dataset generation\n" + << "output " << options.output << '\n' + << "new positions " << options.positions << '\n' + << "depth " << options.depth << '\n' + << "workers " << options.workers << '\n' + << "hash/worker " << options.hashMB << " MB\n" + << "random plies " << options.randomPlies << '\n' + << "first sample " << std::max({1, options.randomPlies, options.skipPlies}) << '\n' + << "sample every " << options.sampleEvery << '\n' + << "exploration " << options.exploration << '\n' + << "explore MultiPV " << options.explorationMultiPV << '\n' + << "explore maxloss " << options.explorationMaxLoss << " cp\n" + << "dedup " << (options.dedup ? "on" : "off") << '\n' + << "seed " << options.seed << "\n\n"; + + std::atomic nextGameId{0}; + std::atomic failed{false}; + GenerationStats stats; + std::vector workers; + workers.reserve(options.workers); + + for (unsigned workerId = 0; workerId < options.workers; ++workerId) { + workers.emplace_back( + workerMain, + workerId, + std::cref(options), + std::ref(writer), + std::ref(deduper), + std::ref(stats), + std::ref(nextGameId), + std::ref(failed)); + } + + for (auto& worker : workers) + worker.join(); + + if (failed.load(std::memory_order_relaxed)) { + std::cerr << "dataset generation failed; record count was not committed\n"; + return 1; + } + + if (!writer.commit(error)) { + std::cerr << "dataset commit failed: " << error << '\n'; + return 1; + } + + std::cout + << "done: added " << writer.addedCount() + << " unique positions; total " << writer.totalCount() + << "; skipped duplicates " << stats.duplicateSamples.load(std::memory_order_relaxed) + << "; explored moves " << stats.exploredMoves.load(std::memory_order_relaxed) + << '\n'; + + return 0; +}