diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d51eaf5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +** +!Dockerfile +!inserter/ +!inserter/** diff --git a/.gitignore b/.gitignore index 9797cdc..5c00bd7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target /pkg +/data/ # Editor directories and files .idea @@ -37,4 +38,4 @@ env.bak/ venv.bak/ # AI agent personal preferences (per-developer, not shared) -.ai/local.md \ No newline at end of file +.ai/local.md diff --git a/Dockerfile b/Dockerfile index 2b9f661..2d921b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,18 @@ -FROM rust:latest AS builder +ARG IMAGE_VERSION=dev +ARG VCS_REF=unknown + +FROM rust:1.97.0@sha256:b92b8c8574f8f3b207fcb0912fb3e2de4041580b5934d90312d53938c9a038a9 AS builder WORKDIR /app COPY ./inserter/ . RUN rustup target add x86_64-unknown-linux-musl -RUN cargo build --release --target x86_64-unknown-linux-musl --bin dsp_seed_finder +RUN cargo build --release --locked --target x86_64-unknown-linux-musl --bin dsp_seed_finder FROM scratch +ARG IMAGE_VERSION +ARG VCS_REF +LABEL org.opencontainers.image.title="DSP Seed Finder" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.revision="${VCS_REF}" \ + org.opencontainers.image.source="https://github.com/SuperB3333/DSPSeedDatabase" COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/dsp_seed_finder /app/app CMD ["/app/app"] diff --git a/README.md b/README.md index 6a534ab..0228019 100644 --- a/README.md +++ b/README.md @@ -1,74 +1,201 @@ # DSP Seed Finder -A high-performance pipeline for generating, indexing, and searching Dyson Sphere Program (DSP) galaxy seeds. It replaces the trial-and-error of seed hunting with a multi-threaded Rust generator and a flexible Python-based SQL search engine. +DSP Seed Finder generates galaxy data for Dyson Sphere Program (DSP). A Rust +program writes the data to PostgreSQL. PostgREST supplies a read-only +application programming interface (API) that uses Hypertext Transfer Protocol +(HTTP). -## Core Capabilities +## Services -- **High-Speed Generation:** Multi-threaded Rust implementation of the DSP galaxy generation algorithm, capable of processing thousands of seeds per second. -- **SQL-Powered Search:** Exports seed data to PostgreSQL for complex queries that the game's UI cannot perform (e.g., "Find a seed with 6 O-stars and a tidally locked planet inside the Dyson radius"). -- **Resume-Safe:** Atomic checkpointing ensures that long-running generation jobs (e.g., processing 10 million seeds) can be paused and resumed without data corruption or duplication. -- **Flexible Scoring:** A Python scoring engine ranks seeds based on weighted metrics like total luminosity, rare ore availability, or "inside-sphere" planet counts. +- `seedfinder` generates stars, planets, gases, and vein amounts. +- `postgres` stores the generated data. +- `api` gives read-only access to the `stars` and `planets` tables. -## Quick Start +The generator processes the configured seed range. The generator then stops. + +## Requirements + +Install these tools: + +- Docker Engine +- Docker Compose +- `curl` for the API check + +The release platform is Linux on a 64-bit x86 processor. + +## Start The Full Stack + +1. Set a PostgreSQL password. + + ```bash + export POSTGRES_PASSWORD='replace-this-password' + ``` + +2. Start the full stack with the default 100-seed range. + + ```bash + docker compose -f compose/compose_full.yaml up -d --wait + ``` + +3. Make sure that the generator exit code is `0`. + + ```bash + docker inspect seedfinder --format '{{.State.ExitCode}}' + ``` + +4. Read one star from the API. + + ```bash + curl --fail 'http://127.0.0.1:3000/stars?select=seed,star_index&limit=1' + ``` + +Set `END_SEED` to the first seed that the generator does not process. -### 1. Infrastructure -The fastest way to get the database ready is via Docker Compose: ```bash -docker compose -f compose/compose.postgres.yaml up -d +END_SEED=1000 docker compose -f compose/compose_full.yaml up -d --wait ``` -### 2. Schema Setup -Initialize the database schema (requires `psycopg2`): +## Start Services Independently + +Set the PostgreSQL password: + ```bash -python3 create_database.py +export POSTGRES_PASSWORD='replace-this-password' ``` -### 3. Generate Seeds -Build and run the generator. This will populate the database with seeds in the specified range. +Start PostgreSQL: + ```bash -# Configuration via environment variables -START_SEED=0 END_SEED=100000 WORKER_THREADS=16 cargo run --release +docker compose -f compose/compose.postgres.yaml up -d --wait ``` -### 4. Search & Score -Find the best seeds based on your own criteria: +Start the generator after the PostgreSQL health check passes: + ```bash -# Example: Rank seeds by O-star luminosity and Silicon availability -python3 score_seeds.py --weight luminosity=1.5 --weight ore_silicium=1.0 --top 10 +POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ +docker compose -f compose/compose.yaml up ``` -## Architecture +The standalone generator uses Linux host networking. It connects to PostgreSQL +at `127.0.0.1:5432`. -### The Pipeline -1. **Generation (`src/`)**: A Rust crate that replicates the game's internal generation logic. It bypasses all rendering/UI and uses `crossbeam-channel` to pipe generated CSV data from worker threads to database writer threads. -2. **Ingestion**: Uses PostgreSQL `COPY FROM STDIN` for maximum throughput. Tables are created as `UNLOGGED` by default to minimize Write-Ahead Log (WAL) overhead. -3. **Search (`rules.py`, `score_seeds.py`)**: A Python layer that compiles high-level search criteria into optimized SQL. +## Configuration -### Database Schema -Data is split into two primary tables: -- **`stars`**: High-level system data (luminosity, type, star-level aggregated ores). -- **`planets`**: Detailed planet-level data (vein counts, tidal locking, orbital radius, gas rates). +The generator reads these environment variables: -**Note on Sentinels:** To handle the variance in planet types, the database uses `-1` as a sentinel value for ores that do not exist on a given planet (e.g., iron on a gas giant). The scoring engine uses `GREATEST(estimate, 0)` to ensure these sentinels never subtract from a seed's score. +| Variable | Executable default | Function | +| --- | ---: | --- | +| `START_SEED` | `0` | Set the first seed. | +| `END_SEED` | `10000` | Set the first seed that the generator does not process. | +| `WORKER_THREADS` | Adaptive | Override the generator worker count. | +| `WRITER_THREADS` | `1` | Set the PostgreSQL writer count. | +| `COMMIT_COUNT` | `64` | Set the maximum seeds in one transaction. | +| `CHANNEL_SIZE` | `64` | Set the completed-seed queue capacity. | +| `BENCHMARK` | Off | Discard generated output. | +| `NO_TUI` | Off | Disable the terminal user interface (TUI). | +| `PG_NETLOC` | `localhost` | Set the PostgreSQL host. | +| `PG_PORT` | `5432` | Set the PostgreSQL port. | +| `PG_DBNAME` | `dsp` | Set the PostgreSQL database. | +| `PG_USER` | `postgres` | Set the PostgreSQL user. | +| `PG_PASS` | `rootpassword` | Set the PostgreSQL password. | -## Configuration +The adaptive worker count is the smaller value of the seed count and 32. Set +`WORKER_THREADS` to select a different worker count. + +The full-stack Compose file sets `END_SEED` to `100`. The standalone generator +Compose file sets `END_SEED` to `1000`. Each generator Compose file enables +`NO_TUI`. -The Rust generator is configured via environment variables: -- `START_SEED` / `END_SEED`: The range of seeds to process. -- `WORKER_THREADS`: Number of threads generating galaxy data. -- `WRITER_THREADS`: Number of threads pushing data to Postgres. -- `CHECKPOINT_FILE`: Path to the progress tracking file (default: `checkpoints.txt`). -- `BENCHMARK=1`: Disables DB writes to test pure generation throughput. +## Build And Test -## Sharp Edges +Build the release image: -- **Checkpoint Invariants:** The generator bails out if it detects a mismatch between the current config and an existing `checkpoints.txt`. If you change your seed range or worker count, you must delete the old checkpoint file. -- **UNLOGGED Tables:** For speed, tables are `UNLOGGED`. If the database crashes, data may be lost. Run `ALTER TABLE ... SET LOGGED` if you require durability. -- **Database Load:** The generator can easily saturate a standard Postgres installation. If you see "channel stall" warnings, try increasing `COMMIT_COUNT` or decreasing `WORKER_THREADS`. +```bash +docker build \ + --build-arg IMAGE_VERSION=0.7.0 \ + --build-arg VCS_REF="$(git rev-parse HEAD)" \ + --tag dsp-seed-finder:0.7.0 . +``` + +Run the Rust tests in a container: + +```bash +docker run --rm \ + --mount type=bind,source="$PWD/inserter",target=/src,readonly \ + --workdir /src \ + --env CARGO_TARGET_DIR=/tmp/target \ + rust:1.97.0@sha256:b92b8c8574f8f3b207fcb0912fb3e2de4041580b5934d90312d53938c9a038a9 \ + cargo test --release --locked +``` -## Making Your First Change +## Performance -The shortest path to a meaningful change is adding a new metric to the scoring engine: -1. Open `score_seeds.py`. -2. Add a new entry to the `_build_metrics` dictionary (e.g., a filter for specific planet themes). -3. Run the script with your new metric: `--weight your_metric=1.0`. +The controlled test used one worker. The test measured the paired change in +throughput. The median gain was 38.0% for version 0.7.0. Median throughput +increased from 2.649 seeds/s to 3.668 seeds/s. The bootstrap interval was 32.0% +to 39.5% at 95% confidence. + +For the method, raw data, discarded experiments, and scaling results, refer to +[`docs/performance.md`](docs/performance.md). + +## Determinism + +A determinism test ran the same version 0.7.0 executable two times. The test +processed seeds from 0 to 999. The `cmp` tool found no different bytes. Each +file contained 50,069,759 bytes. + +```text +d188c78a555c95f2188db893a8c9890ec2a50848c9b8826f6e72c7104b3b7acd +``` + +The value is a 256-bit Secure Hash Algorithm (SHA-256) digest. The result is +valid for the tested executable. GNU and musl math libraries can give different +planet values. + +## Data Storage + +The database uses PostgreSQL `UNLOGGED` tables. PostgreSQL does not restore data +from these tables after a crash. Use logged tables if crash recovery is +necessary. + +The database files are in `data/`. + +1. Make sure that the current directory is the repository root. + + ```bash + test -f Dockerfile + ``` + +2. Stop all services. + + ```bash + docker compose -f compose/compose_full.yaml down --remove-orphans + ``` + +3. Remove the database directory. + + ```bash + sudo rm -rf -- data + ``` + +4. Create an empty database directory. + + ```bash + mkdir data + ``` + +## Release Image + +The release image is `toti330/dsp_seed_finder:0.7.0`. + +The published image has this digest: + +```text +sha256:58d968aa4a6661b7b203a6716bd0de7c5a39763d688410b1d600d617513181ca +``` + +Examine the remote image: + +```bash +docker buildx imagetools inspect toti330/dsp_seed_finder:0.7.0 +``` diff --git a/api/main.py b/api/main.py deleted file mode 100644 index ae7a34e..0000000 --- a/api/main.py +++ /dev/null @@ -1,30 +0,0 @@ -from fastapi import FastAPI -from hashlib import sha256 - -from query_database import query_database - -import time - -app = FastAPI() - -glob_return_dict = {} - -@app.post("/start_query") -def start_query(query: str, params: list): - h = sha256(query.encode()) - h.update(bytes(int(time.time()))) - qid = h.hexdigest() - glob_return_dict[qid] = None - - query_database((query, params), qid) - - return qid - -@app.post("/query_ready") -def query_status(qid: str): - return "true" if glob_return_dict[qid] is not None else "false" -@app.post("/get_result") -def query_result(qid: str): - res = list(glob_return_dict[qid]) - glob_return_dict[qid] = None - return res diff --git a/api/query_database.py b/api/query_database.py deleted file mode 100644 index 8ab55b4..0000000 --- a/api/query_database.py +++ /dev/null @@ -1,23 +0,0 @@ -import threading, psycopg2, os - - - -def _new_connection(): - return psycopg2.connect(database=os.getenv("PG_DBNAME"), user=os.getenv("PG_USER"), password=os.getenv("PG_PASS"), host=os.getenv("PG_NETLOC"), port=os.getenv("PG_PORT")) - -_client = None -def _get_client(): - global _client - if _client is not None: return _client - _client = _new_connection() - return _client - -def query_database(query: tuple[str, list], qid: str) -> None: - threading.Thread(target=_query_database, args=(query + "\nLIMIT 100", qid), name=f"Database query {qid}", daemon=True) -def _query_database(query: tuple[str, list], qid: str) -> None: - global glob_return_dict - cur = _get_client().cursor() - cur.execute(*query) - res = cur.fetchall() - - glob_return_dict[qid] = map(lambda x: x[0], res) \ No newline at end of file diff --git a/api/requirements.txt b/api/requirements.txt deleted file mode 100644 index 6270e8f..0000000 --- a/api/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -fastapi -uvicorn -python-multipart -psycopg2 \ No newline at end of file diff --git a/benchmarks/results/2026-07-29-experiments.json b/benchmarks/results/2026-07-29-experiments.json new file mode 100644 index 0000000..2a4dd3a --- /dev/null +++ b/benchmarks/results/2026-07-29-experiments.json @@ -0,0 +1,171 @@ +{ + "schema_version": 1, + "date": "2026-07-29", + "baseline_commit": "c0ed162ddf820b8d4ada0127b597cd64a8e35a88", + "platform": "linux/amd64", + "rust_version": "1.97.0", + "system": { + "cpu": "Intel Xeon Platinum 8370C at 2.80 GHz", + "logical_processors": 2, + "physical_cores": 1, + "memory_gib": 7.7, + "kernel": "Linux 6.17.0-1021-azure", + "docker_version": "29.6.1" + }, + "profile": { + "baseline_sample_percent": { + "simplex_noise_3d": 45.39, + "page_fault_handling": 15.55, + "terrain_height_interpolation": 10.56 + }, + "optimized_sample_percent": { + "simplex_noise_3d": 56.2, + "terrain_height_interpolation": 10.98 + }, + "baseline_100_seed_minor_page_faults": 3319980, + "pooled_cache_100_seed_minor_page_faults": 7619, + "hardware_counters_supported": false + }, + "experiments": [ + { + "name": "pooled-terrain-cache", + "decision": "keep", + "median_change_percent": 27.1381, + "bootstrap_95_percent_ci": [16.5794, 31.3509], + "output_equal": true + }, + { + "name": "spacing-before-terrain", + "decision": "keep", + "median_change_percent": 5.1378, + "bootstrap_95_percent_ci": [2.6168, 8.3876], + "output_equal": true + }, + { + "name": "compact-permutations", + "decision": "keep", + "median_change_percent": 2.9471, + "bootstrap_95_percent_ci": [0.9125, 7.5649], + "output_equal": true + }, + { + "name": "fused-algorithm-12-noise", + "decision": "revert", + "median_change_percent": 0.1962, + "bootstrap_95_percent_ci": [-4.7622, 0.7467], + "output_equal": true + }, + { + "name": "reuse-algorithm-9-noise", + "decision": "keep", + "median_change_percent": 0.7536, + "bootstrap_95_percent_ci": [-5.1496, 4.9991], + "median_user_cpu_change_percent": 0.9846, + "output_equal": true + }, + { + "name": "conditional-algorithm-10-noise", + "decision": "revert", + "median_change_percent": -0.9184, + "bootstrap_95_percent_ci": [-2.9559, 4.2627], + "output_equal": true + }, + { + "name": "conditional-algorithm-5-and-6-noise", + "decision": "keep", + "median_change_percent": 0.3772, + "bootstrap_95_percent_ci": [-1.6933, 7.9741], + "median_user_cpu_change_percent": 1.1884, + "output_equal": true + }, + { + "name": "partial-algorithm-4-distance", + "decision": "revert", + "median_change_percent": -1.8394, + "bootstrap_95_percent_ci": [-8.9872, -0.1795], + "output_equal": true + }, + { + "name": "blocking-writer-input", + "decision": "keep-for-reliability", + "median_change_percent": -1.4859, + "bootstrap_95_percent_ci": [-6.7233, 0.8698] + }, + { + "name": "small-pipeline-defaults", + "decision": "keep", + "baseline_seeds_per_second": 6.5683, + "candidate_seeds_per_second": 6.5949, + "seed_count": 500 + }, + { + "name": "adaptive-worker-count", + "decision": "keep", + "baseline_workers": 8, + "candidate_workers": 32, + "baseline_seeds_per_second": 5.9618, + "candidate_seeds_per_second": 6.5949, + "seed_count": 500 + }, + { + "name": "final-one-worker-comparison", + "decision": "keep", + "median_change_percent": 37.9963, + "bootstrap_95_percent_ci": [31.9682, 39.4965], + "seed_count": 100 + } + ], + "database_scaling": { + "seed_count": 500, + "postgres_version": "16", + "writer_count": 1, + "commit_count": 64, + "samples": [ + {"workers": 2, "seeds_per_second": 4.538395, "max_rss_kib": 14432}, + {"workers": 8, "seeds_per_second": 5.961821, "max_rss_kib": 18008}, + {"workers": 16, "seeds_per_second": 6.179457, "max_rss_kib": 23876}, + {"workers": 32, "seeds_per_second": 6.594936, "max_rss_kib": 33808} + ], + "four_writer_seeds_per_second": 6.589842, + "baseline_scale_status": "failed after writer sink timeout" + }, + "correctness": { + "pre_copy_seed_range": [0, 100], + "pre_copy_sha256": "3f26eb1b456d5a7b71f40a98ab03e0472b05e41cc31e65740dca002da6afa7d6", + "copy_acceptance": { + "seed_count": 10, + "batch_size": 3, + "star_rows": 640, + "planet_rows": 2446 + }, + "final_repeat": { + "seed_range": [0, 1000], + "runs": 2, + "bytes": 50069759, + "sha256": "d188c78a555c95f2188db893a8c9890ec2a50848c9b8826f6e72c7104b3b7acd" + }, + "database_worker_comparison": { + "seed_range": [0, 500], + "worker_counts": [8, 32], + "star_rows": 32000, + "planet_rows": 122266, + "star_hash": "51ddf9bbe5e3a6dbf17dfa5e65f89621", + "planet_hash": "32096b2ffd8476930b437c1b9711cb50" + }, + "gnu_and_musl_planet_hashes_equal": false + }, + "not_tested": [ + { + "name": "unsafe-fast-math-or-simd", + "reason": "Can change terrain comparisons and random-number consumption" + }, + { + "name": "profile-guided-optimization", + "reason": "Binds code layout to one training corpus and hardware profile" + }, + { + "name": "dynamic-seed-queue", + "reason": "Requires a new checkpoint protocol for in-progress seeds" + } + ] +} diff --git a/benchmarks/results/2026-07-29-final.json b/benchmarks/results/2026-07-29-final.json new file mode 100644 index 0000000..ca86fa2 --- /dev/null +++ b/benchmarks/results/2026-07-29-final.json @@ -0,0 +1,223 @@ +{ + "schema_version": 1, + "source": { + "baseline_commit": "c0ed162ddf820b8d4ada0127b597cd64a8e35a88", + "optimized_behavior_commit": "1b72146fb52ecf0ab01e4f1b1ff0a6fb05637983", + "rust_version": "1.97.0", + "target": "x86_64-unknown-linux-musl" + }, + "created_utc": "2026-07-29T18:53:03.345430+00:00", + "host": { + "platform": "Linux-6.17.0-1021-azure-x86_64-with-glibc2.39", + "processor_count": 2, + "cpu_set": "0" + }, + "workload": { + "start_seed": 0, + "end_seed": 100, + "seed_count": 100, + "workers": 1, + "writer_sinks": 1, + "channel_size": 64 + }, + "executables": { + "baseline": { + "path": "/tmp/opencode/dsp-optimization/baseline/dsp_seed_finder", + "sha256": "74e96f68daac80a35d4296fd06204d8e4c393785d493c5414ff5679d56f5747c" + }, + "optimized-0.7.0": { + "path": "/tmp/opencode/dsp-optimization/target-final/x86_64-unknown-linux-musl/release/dsp_seed_finder", + "sha256": "b1d3c5d7216848b9fd73cccc36d6e9142574d8ac5d9b0e1e4dbeed4fcc83a31a" + } + }, + "results": [ + { + "sequence": 0, + "variant": "baseline", + "throughput_seeds_per_second": 2.629865, + "wall_seconds": 38.02, + "user_seconds": 28.65, + "system_seconds": 6.75, + "max_rss_kib": 5980, + "minor_page_faults": 3320208, + "major_page_faults": 0, + "voluntary_context_switches": 486, + "involuntary_context_switches": 20167 + }, + { + "sequence": 1, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.669744, + "wall_seconds": 27.25, + "user_seconds": 25.71, + "system_seconds": 0.13, + "max_rss_kib": 5696, + "minor_page_faults": 7036, + "major_page_faults": 0, + "voluntary_context_switches": 377, + "involuntary_context_switches": 7864 + }, + { + "sequence": 2, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.666149, + "wall_seconds": 27.28, + "user_seconds": 25.73, + "system_seconds": 0.1, + "max_rss_kib": 5632, + "minor_page_faults": 7036, + "major_page_faults": 0, + "voluntary_context_switches": 377, + "involuntary_context_switches": 6891 + }, + { + "sequence": 3, + "variant": "baseline", + "throughput_seeds_per_second": 2.648636, + "wall_seconds": 37.75, + "user_seconds": 28.68, + "system_seconds": 6.65, + "max_rss_kib": 5952, + "minor_page_faults": 3320918, + "major_page_faults": 0, + "voluntary_context_switches": 485, + "involuntary_context_switches": 19807 + }, + { + "sequence": 4, + "variant": "baseline", + "throughput_seeds_per_second": 2.648638, + "wall_seconds": 37.75, + "user_seconds": 28.53, + "system_seconds": 6.75, + "max_rss_kib": 5964, + "minor_page_faults": 3320968, + "major_page_faults": 0, + "voluntary_context_switches": 482, + "involuntary_context_switches": 19130 + }, + { + "sequence": 5, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.643891, + "wall_seconds": 27.44, + "user_seconds": 25.64, + "system_seconds": 0.1, + "max_rss_kib": 5676, + "minor_page_faults": 6988, + "major_page_faults": 0, + "voluntary_context_switches": 381, + "involuntary_context_switches": 8187 + }, + { + "sequence": 6, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.68346, + "wall_seconds": 27.15, + "user_seconds": 25.64, + "system_seconds": 0.11, + "max_rss_kib": 5632, + "minor_page_faults": 6860, + "major_page_faults": 0, + "voluntary_context_switches": 380, + "involuntary_context_switches": 7356 + }, + { + "sequence": 7, + "variant": "baseline", + "throughput_seeds_per_second": 2.680183, + "wall_seconds": 37.31, + "user_seconds": 28.66, + "system_seconds": 6.75, + "max_rss_kib": 5976, + "minor_page_faults": 3320980, + "major_page_faults": 0, + "voluntary_context_switches": 478, + "involuntary_context_switches": 24800 + }, + { + "sequence": 8, + "variant": "baseline", + "throughput_seeds_per_second": 2.622841, + "wall_seconds": 38.13, + "user_seconds": 28.66, + "system_seconds": 6.71, + "max_rss_kib": 6008, + "minor_page_faults": 3320999, + "major_page_faults": 0, + "voluntary_context_switches": 489, + "involuntary_context_switches": 22307 + }, + { + "sequence": 9, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.657601, + "wall_seconds": 27.34, + "user_seconds": 25.7, + "system_seconds": 0.12, + "max_rss_kib": 5696, + "minor_page_faults": 7037, + "major_page_faults": 0, + "voluntary_context_switches": 378, + "involuntary_context_switches": 8700 + }, + { + "sequence": 10, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.694465, + "wall_seconds": 27.07, + "user_seconds": 25.66, + "system_seconds": 0.11, + "max_rss_kib": 5620, + "minor_page_faults": 6970, + "major_page_faults": 0, + "voluntary_context_switches": 379, + "involuntary_context_switches": 8191 + }, + { + "sequence": 11, + "variant": "baseline", + "throughput_seeds_per_second": 2.920451, + "wall_seconds": 34.24, + "user_seconds": 27.5, + "system_seconds": 5.93, + "max_rss_kib": 5920, + "minor_page_faults": 3320262, + "major_page_faults": 0, + "voluntary_context_switches": 449, + "involuntary_context_switches": 6406 + } + ], + "summary": { + "throughput": { + "baseline": { + "median": 2.648637, + "mad": 0.02228399999999975, + "p25": 2.63455775, + "p75": 2.67229675, + "minimum": 2.622841, + "maximum": 2.920451 + }, + "optimized-0.7.0": { + "median": 3.6679465, + "mad": 0.012929500000000038, + "p25": 3.659738, + "p75": 3.680031, + "minimum": 3.643891, + "maximum": 3.694465 + } + }, + "paired_change_percent": { + "median": 37.99626012180245, + "mad": 1.0093619576554183, + "p25": 37.4688720746932, + "p75": 39.193030250875104, + "minimum": 26.50323528797436, + "maximum": 39.54115515435204, + "bootstrap_95_percent_ci": [ + 31.968194086715165, + 39.496515977559426 + ] + } + } +} diff --git a/benchmarks/run_comparison.py b/benchmarks/run_comparison.py new file mode 100755 index 0000000..f3c8ddd --- /dev/null +++ b/benchmarks/run_comparison.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Compare two inserter executables with controlled process runs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import random +import re +import statistics +import subprocess +import tempfile +from datetime import datetime, timezone +from pathlib import Path + + +THROUGHPUT_PATTERN = re.compile(r"^seeds/sec: ([0-9.]+)$", re.MULTILINE) +TIME_FIELDS = ( + "wall_seconds", + "user_seconds", + "system_seconds", + "max_rss_kib", + "minor_page_faults", + "major_page_faults", + "voluntary_context_switches", + "involuntary_context_switches", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--candidate", required=True, type=Path) + parser.add_argument("--candidate-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--start-seed", type=int, default=0) + parser.add_argument("--end-seed", type=int, default=60) + parser.add_argument("--workers", type=int, default=1) + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--cpus", default="0") + return parser.parse_args() + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def percentile(values: list[float], fraction: float) -> float: + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + weight = position - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def describe(values: list[float]) -> dict[str, float]: + median = statistics.median(values) + deviations = [abs(value - median) for value in values] + return { + "median": median, + "mad": statistics.median(deviations), + "p25": percentile(values, 0.25), + "p75": percentile(values, 0.75), + "minimum": min(values), + "maximum": max(values), + } + + +def bootstrap_ci(values: list[float], samples: int = 20_000) -> list[float]: + generator = random.Random(0) + medians = [] + for _ in range(samples): + draw = [generator.choice(values) for _ in values] + medians.append(statistics.median(draw)) + return [percentile(medians, 0.025), percentile(medians, 0.975)] + + +def run_once( + label: str, + executable: Path, + args: argparse.Namespace, + sequence: int, +) -> dict[str, object]: + environment = os.environ.copy() + environment.update( + { + "BENCHMARK": "1", + "START_SEED": str(args.start_seed), + "END_SEED": str(args.end_seed), + "WORKER_THREADS": str(args.workers), + "WRITER_THREADS": "1", + "CHANNEL_SIZE": "64", + "COMMIT_COUNT": "64", + "NO_TUI": "1", + "LOG_LEVEL": "error", + "LOG_INTERVAL": "none", + "CHECKPOINT_INTERVAL": "none", + "OVERRIDE_CHECKPOINTS": "1", + "CHECKPOINT_FILE": "/tmp/dsp-benchmark-no-checkpoint", + "PG_NETLOC": "127.0.0.1", + "PG_PORT": "1", + } + ) + + with tempfile.NamedTemporaryFile() as time_output: + command = [ + "/usr/bin/time", + "-f", + "%e\n%U\n%S\n%M\n%R\n%F\n%w\n%c", + "-o", + time_output.name, + "taskset", + "-c", + args.cpus, + str(executable), + ] + completed = subprocess.run( + command, + env=environment, + check=False, + text=True, + capture_output=True, + ) + time_output.seek(0) + measurements = time_output.read().decode("ascii").splitlines() + + if completed.returncode != 0: + raise RuntimeError( + f"{label} failed with status {completed.returncode}:\n{completed.stderr}" + ) + match = THROUGHPUT_PATTERN.search(completed.stdout) + if match is None or len(measurements) != len(TIME_FIELDS): + raise RuntimeError(f"Cannot parse {label} output:\n{completed.stdout}") + + values = dict(zip(TIME_FIELDS, measurements, strict=True)) + integer_fields = set(TIME_FIELDS[3:]) + result: dict[str, object] = { + "sequence": sequence, + "variant": label, + "throughput_seeds_per_second": float(match.group(1)), + } + for field, value in values.items(): + result[field] = int(value) if field in integer_fields else float(value) + return result + + +def summarize(results: list[dict[str, object]], candidate_name: str) -> dict[str, object]: + by_variant = { + label: [ + float(result["throughput_seeds_per_second"]) + for result in results + if result["variant"] == label + ] + for label in ("baseline", candidate_name) + } + baseline = by_variant["baseline"] + candidate = by_variant[candidate_name] + paired_changes = [ + (new / old - 1.0) * 100.0 for old, new in zip(baseline, candidate, strict=True) + ] + return { + "throughput": { + "baseline": describe(baseline), + candidate_name: describe(candidate), + }, + "paired_change_percent": { + **describe(paired_changes), + "bootstrap_95_percent_ci": bootstrap_ci(paired_changes), + }, + } + + +def main() -> None: + args = parse_args() + for executable in (args.baseline, args.candidate): + if not executable.is_file(): + raise SystemExit(f"Executable does not exist: {executable}") + + variants = { + "baseline": args.baseline.resolve(), + args.candidate_name: args.candidate.resolve(), + } + order = ["baseline", args.candidate_name, args.candidate_name, "baseline"] + results = [] + for _ in range(args.rounds): + for label in order: + result = run_once(label, variants[label], args, len(results)) + results.append(result) + print( + f"{label}: {result['throughput_seeds_per_second']:.6f} seeds/s", + flush=True, + ) + + artifact = { + "schema_version": 1, + "created_utc": datetime.now(timezone.utc).isoformat(), + "host": { + "platform": platform.platform(), + "processor_count": os.cpu_count(), + "cpu_set": args.cpus, + }, + "workload": { + "start_seed": args.start_seed, + "end_seed": args.end_seed, + "seed_count": args.end_seed - args.start_seed, + "workers": args.workers, + "writer_sinks": 1, + "channel_size": 64, + }, + "executables": { + label: {"path": str(path), "sha256": file_sha256(path)} + for label, path in variants.items() + }, + "results": results, + "summary": summarize(results, args.candidate_name), + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(artifact, indent=2) + "\n", encoding="ascii") + print(json.dumps(artifact["summary"], indent=2)) + + +if __name__ == "__main__": + main() diff --git a/compose/compose.md b/compose/compose.md index d0647b8..b7bf69a 100644 --- a/compose/compose.md +++ b/compose/compose.md @@ -1,30 +1,75 @@ -# compose -This folder contains Docker Compose configurations for running project services -individually. - -#### Variations -- **compose.yaml** -Runs only the `seedfinder` container. It is configured to connect to an -individually managed PostgreSQL instance at `host.docker.internal:5432` using -database `dsp`, user `postgres`, and password `rootpassword`. It uses Docker's -built-in `bridge` network so `host-gateway` reaches PostgreSQL's published port -without joining PostgreSQL's Compose network. - -- **compose.postgres.yaml** -Runs PostgreSQL and PostgREST. PostgreSQL is available for individual connections from -local tools or from `seedfinder` in `compose.yaml`. It publishes PostgreSQL on localhost -port `5432`, creates database `dsp`, and persists data in `../data`. PostgREST is -exposed on port `3000` and provides a RESTful API over the `public` schema. - -#### Usage -- Start PostgreSQL + PostgREST: - `docker compose -f compose.postgres.yaml up -d` - -- Start seedfinder only: - `docker compose -f compose.yaml up -d` - -- Connect locally (PostgreSQL): - `postgresql://postgres:rootpassword@localhost:5432/dsp` - -- PostgREST API: - `http://localhost:3000` +# Docker Compose Files + +Run all commands in this document from the repository root. + +## Full Stack + +`compose/compose_full.yaml` uses this startup sequence: + +1. Docker starts PostgreSQL. +2. Docker waits for the PostgreSQL health check to pass. +3. The generator creates the schema. +4. The generator writes the configured seeds. +5. PostgREST starts after the generator exits with code `0`. + +Set the PostgreSQL password: + +```bash +export POSTGRES_PASSWORD='replace-this-password' +``` + +Start the full stack: + +```bash +docker compose -f compose/compose_full.yaml up -d --wait +``` + +The default range contains seeds from 0 to 99. The application programming +interface (API) listens on `127.0.0.1:3000`. + +## PostgreSQL Only + +`compose/compose.postgres.yaml` starts PostgreSQL. PostgreSQL stores data in +`data/`. PostgreSQL listens on `127.0.0.1:5432` by default. + +Set the PostgreSQL password: + +```bash +export POSTGRES_PASSWORD='replace-this-password' +``` + +Start PostgreSQL: + +```bash +docker compose -f compose/compose.postgres.yaml up -d --wait +``` + +Set `POSTGRES_PORT` to use a different host port. + +## Generator Only + +`compose/compose.yaml` starts one finite generator job. The generator uses Linux +host networking. It connects to PostgreSQL at `127.0.0.1:5432`. + +Set the PostgreSQL password: + +```bash +export POSTGRES_PASSWORD='replace-this-password' +``` + +Start the generator: + +```bash +docker compose -f compose/compose.yaml up +``` + +Set `START_SEED` and `END_SEED` before the command to select a range. + +## Stop Services + +```bash +docker compose -f compose/compose_full.yaml down --remove-orphans +``` + +The `docker compose down` command does not remove the bind-mounted database +files in `data/`. diff --git a/compose/compose.postgres.yaml b/compose/compose.postgres.yaml index 96fc88d..a9cc8fb 100644 --- a/compose/compose.postgres.yaml +++ b/compose/compose.postgres.yaml @@ -1,18 +1,18 @@ services: postgres: - image: postgres:16 + image: postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 container_name: postgres environment: POSTGRES_USER: postgres - POSTGRES_PASSWORD: rootpassword + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} POSTGRES_DB: dsp volumes: - "../data:/var/lib/postgresql/data" ports: - - "5432:5432" + - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" restart: unless-stopped healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] interval: 5s timeout: 5s retries: 5 diff --git a/compose/compose.yaml b/compose/compose.yaml index ad62536..4b6574d 100644 --- a/compose/compose.yaml +++ b/compose/compose.yaml @@ -1,17 +1,16 @@ services: seedfinder: - image: toti330/dsp_seed_finder:0.6.0 + image: toti330/dsp_seed_finder:0.7.0@sha256:58d968aa4a6661b7b203a6716bd0de7c5a39763d688410b1d600d617513181ca container_name: seedfinder environment: - END_SEED: 1000 # max is 10,000,000 (10m) + START_SEED: ${START_SEED:-0} + END_SEED: ${END_SEED:-1000} NO_TUI: 1 PG_USER: postgres - PG_PASS: rootpassword # Default values + PG_PASS: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} PG_PORT: 5432 PG_DBNAME: dsp - PG_NETLOC: host.docker.internal - LOG_LEVEL: debug - network_mode: bridge - restart: on-failure - extra_hosts: - - "host.docker.internal:host-gateway" + PG_NETLOC: 127.0.0.1 + LOG_LEVEL: ${LOG_LEVEL:-info} + network_mode: host + restart: "no" diff --git a/compose/compose_full.yaml b/compose/compose_full.yaml index a650379..7bb1031 100644 --- a/compose/compose_full.yaml +++ b/compose/compose_full.yaml @@ -1,39 +1,48 @@ services: postgres: - image: postgres:16 + image: postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 container_name: postgres environment: POSTGRES_USER: postgres - POSTGRES_PASSWORD: rootpassword + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} POSTGRES_DB: dsp volumes: - "../data:/var/lib/postgresql/data" ports: - - "5432:5432" + - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" restart: unless-stopped healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] interval: 5s timeout: 5s retries: 5 seedfinder: - image: toti330/seedfinder:0.6.0 + image: toti330/dsp_seed_finder:0.7.0@sha256:58d968aa4a6661b7b203a6716bd0de7c5a39763d688410b1d600d617513181ca container_name: seedfinder environment: - END_SEED: 100000 # max is 10,000,000 (10m) - NO_TUI: 1 # Disable progress tui + START_SEED: ${START_SEED:-0} + END_SEED: ${END_SEED:-100} + NO_TUI: 1 PG_USER: postgres - PG_PASS: rootpassword # Default values + PG_PASS: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} PG_PORT: 5432 PG_DBNAME: dsp - PG_NETLOC: host.docker.internal - restart: on-failure + PG_NETLOC: postgres + LOG_LEVEL: ${LOG_LEVEL:-info} + restart: "no" depends_on: - - postgres + postgres: + condition: service_healthy api: - image: superb3333/postgres_api + image: postgrest/postgrest:v12.2.12@sha256:5f4ce744539bbba786b4e24dbbd95bdb2a956dcf568c5374995a0ff4a68f5bd2 container_name: api environment: - PG_DBNAME: dsp - PG_USER: api_user - PG_PASS: api_password + PGRST_DB_URI: postgres://api_user:api_password@postgres:5432/dsp + PGRST_DB_SCHEMA: public + PGRST_DB_ANON_ROLE: api_user + ports: + - "127.0.0.1:${API_PORT:-3000}:3000" + restart: unless-stopped + depends_on: + seedfinder: + condition: service_completed_successfully diff --git a/compose/init/01-create-postgrest-user.sql b/compose/init/01-create-postgrest-user.sql deleted file mode 100644 index ae99b50..0000000 --- a/compose/init/01-create-postgrest-user.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE USER postgrest_api WITH PASSWORD 'postgrest_api_password'; - -GRANT USAGE ON SCHEMA public TO postgrest_api; -GRANT SELECT ON stars TO postgrest_api; -GRANT SELECT ON planets TO postgrest_api; diff --git a/docs/performance.md b/docs/performance.md new file mode 100644 index 0000000..830cf8b --- /dev/null +++ b/docs/performance.md @@ -0,0 +1,206 @@ +# Generation Pipeline Performance + +## Purpose + +This document gives the version 0.7.0 performance results, test method, and +recorded optimization experiments. + +Throughput is the primary performance measure. Wall time is elapsed clock time. +The user central processing unit (CPU) time and the system CPU time measure +processor use. +Minor page faults count memory accesses that the operating system resolves +without disk access. Peak resident set size (RSS) measures peak physical memory +use. + +The test did not measure power. The virtual machine does not supply a stable +power counter. The test did not collect hardware performance-counter data. + +## Test System + +| Item | Value | +| --- | --- | +| Date | 2026-07-29 | +| Processor | Intel Xeon Platinum 8370C at 2.80 GHz | +| Available processors | 2 logical processors, 1 physical core | +| Memory | 7.7 GiB | +| Kernel | Linux 6.17.0-1021-azure | +| Docker | 29.6.1 | +| Rust | 1.97.0 | +| Target | `x86_64-unknown-linux-musl` | +| Baseline source | `c0ed162ddf820b8d4ada0127b597cd64a8e35a88` | +| Baseline executable | `74e96f68daac80a35d4296fd06204d8e4c393785d493c5414ff5679d56f5747c` | +| Optimized executable | `b1d3c5d7216848b9fd73cccc36d6e9142574d8ac5d9b0e1e4dbeed4fcc83a31a` | + +The executable values are 256-bit Secure Hash Algorithm (SHA-256) digests. + +## Controlled Method + +The controlled comparison processed seeds from 0 to 99. Each process used one +generator worker and one output sink. The output sink discarded generated +data. The process ran on logical processor 0. The test disabled logs, +checkpoints, and the terminal user interface (TUI). The process did not read +from or write to a database. + +The benchmark measured executable run time only. It did not measure build time. +It ran the executable sequence `A, B, B, A` three times. This sequence gave six +baseline samples and six optimized samples. + +The script calculates the median and median absolute deviation (MAD). The script +uses 20,000 bootstrap samples and bootstrap random seed 0. The script calculates +a 95% confidence interval for the paired change in median throughput. + +Run the comparison with this command: + +```bash +python3 benchmarks/run_comparison.py \ + --baseline /path/to/baseline/dsp_seed_finder \ + --candidate /path/to/optimized/dsp_seed_finder \ + --candidate-name optimized-0.7.0 \ + --output benchmarks/results/local.json \ + --start-seed 0 \ + --end-seed 100 \ + --workers 1 \ + --rounds 3 \ + --cpus 0 +``` + +The script requires GNU `time` and `taskset`. + +## Result + +| Measure | Baseline median | Version 0.7.0 median | Change | +| --- | ---: | ---: | ---: | +| Throughput | 2.649 seeds/s | 3.668 seeds/s | +38.5% | +| Wall time for 100 seeds | 37.750 s | 27.265 s | -27.8% | +| User CPU time | 28.655 s | 25.680 s | -10.4% | +| System CPU time | 6.730 s | 0.110 s | -98.4% | +| Minor page faults | 3,320,943 | 7,012 | -99.8% | +| Peak RSS | 5,970 KiB | 5,654 KiB | -5.3% | + +The paired throughput gain is 38.0%. Its bootstrap 95% interval is 32.0% to +39.5%. The throughput MAD is 0.022 seeds/s for the baseline and 0.013 seeds/s +for version 0.7.0. + +The machine-readable raw result is +[`benchmarks/results/2026-07-29-final.json`](../benchmarks/results/2026-07-29-final.json). + +## Profile Result + +The baseline software profile had these sample percentages: + +- 45.4% to three-dimensional simplex noise. +- 15.6% to page-fault handling. +- 10.6% to terrain-height interpolation. + +The initial 100-seed run caused 3,319,980 minor page faults. The pooled terrain +cache reduced this value to 7,619 in the first cache experiment. + +After the memory changes, 56.2% of samples occurred in simplex noise. A total +of 11.0% of samples occurred in terrain interpolation. + +The virtual machine did not supply cycles, instructions, branches, or cache +misses from `perf`. The test used software sampling and wall measurements. + +## Optimization Decisions + +| Experiment | Decision | Measured result | +| --- | --- | --- | +| Reuse a compact terrain cache for each worker | Keep | +27.14% median throughput. The 95% interval was +16.58% to +31.35%. | +| Compare vein spacing before terrain | Keep | +5.14%. The interval was +2.62% to +8.39%. | +| Store simplex permutations as `u8` | Keep | +2.95%. The interval was +0.91% to +7.56%. | +| Fuse algorithm 12 noise passes | Revert | +0.20%. The interval was -4.76% to +0.75%. | +| Reuse the duplicate noise result in algorithm 9 | Keep | +0.75% median throughput. The 95% interval was -5.15% to +5.00%. | +| Make algorithm 10 turbulence conditional | Revert | -0.92%. The interval was -2.96% to +4.26%. | +| Do not calculate unused detail noise in algorithms 5 and 6 | Keep | +0.38% median throughput. The 95% interval was -1.69% to +7.97%. | +| Reject algorithm 4 craters with partial distance | Revert | -1.84%. The interval was -8.99% to -0.18%. | +| Wait for the first writer item | Keep for reliability | -1.49% median throughput. The 95% interval was -6.72% to +0.87%. | +| Reduce the default writer count and queue capacity | Keep | +0.40% throughput on 500 seeds. | +| Increase the adaptive worker default | Keep | On 500 seeds, 32 workers had 10.6% higher throughput than 8 workers. | + +The machine-readable decision list is +[`benchmarks/results/2026-07-29-experiments.json`](../benchmarks/results/2026-07-29-experiments.json). + +## PostgreSQL Scaling + +PostgreSQL accepted the binary COPY data in the database test. PostgreSQL 16 +and the generator shared the two logical processors. One writer used batches +of 64 seeds. + +| Workers | Seeds | Throughput | Peak generator RSS | +| ---: | ---: | ---: | ---: | +| 2 | 500 | 4.538 seeds/s | 14,432 KiB | +| 8 | 500 | 5.962 seeds/s | 18,008 KiB | +| 16 | 500 | 6.179 seeds/s | 23,876 KiB | +| 32 | 500 | 6.595 seeds/s | 33,808 KiB | + +The generation time is different for each static seed range. More workers +decrease the idle time at the end of the test. Four database writers did not +increase the 32-worker result. One writer is the default. + +The baseline scale test processed 500 seeds. The test failed because the output +sink treated one second without input as a fatal error. Version 0.7.0 waits for +the first item. Version 0.7.0 writes a non-full batch after one second without +input. + +## Correctness And Determinism + +All eight experiments with an output comparison have equal output. Before the +COPY correction, the test compared seeds from 0 to 99. Each file had this +SHA-256 digest: + +```text +3f26eb1b456d5a7b71f40a98ab03e0472b05e41cc31e65740dca002da6afa7d6 +``` + +The COPY correction changed the byte format. The baseline format did not +include field lengths. The baseline also put one header and one trailer in each +seed buffer. PostgreSQL rejects that format as one batch. + +Version 0.7.0 also corrects two moon values. In version 0.7.0, `sun_distance` +and `inside_ds` use the distance from the moon's parent to the star. The +baseline code used the small moon orbital offset. + +PostgreSQL 16 accepted the corrected COPY stream. A ten-seed test inserted 640 +stars and 2,446 planets. The test used three seeds per batch. The last batch +contained fewer than three seeds. + +Two release tests generated seeds from 0 to 999. The files were byte equal. +Each file contained 50,069,759 bytes and had this SHA-256 digest: + +```text +d188c78a555c95f2188db893a8c9890ec2a50848c9b8826f6e72c7104b3b7acd +``` + +The same release executable wrote seeds from 0 to 499 with 8 workers and with +32 workers. Each run inserted 32,000 stars and 122,266 planets. Sorted database +content had these hashes: + +```text +stars: 51ddf9bbe5e3a6dbf17dfa5e65f89621 +planets: 32096b2ffd8476930b437c1b9711cb50 +``` + +The result is valid for one compiled executable. A GNU build and a musl build +gave different planet hashes. Galaxy generation uses trigonometric and power +functions. Different math libraries can round these functions differently. +The release uses the pinned musl executable. + +## No More Changes + +Version 0.7.0 includes the changes marked `Keep` or `Keep for reliability` in +the optimization table. Simplex noise has the largest profile percentage. + +The test did not enable unsafe fast math. Fast math can change terrain +comparisons and random-number consumption. The test did not add hand-written +single instruction, multiple data (SIMD) code. SIMD code can also change +floating-point results. + +The test did not use profile-guided optimization (PGO). PGO can bind code +layout to one training corpus and one hardware profile. + +A dynamic seed queue can reduce range imbalance with fewer workers. A dynamic +seed queue also changes checkpoint rules for seeds that are in progress. +Version 0.7.0 uses more static ranges. + +The tests stopped after the recorded experiments. More low-level changes can +change floating-point results or checkpoint rules. diff --git a/inserter/Cargo.lock b/inserter/Cargo.lock index dec3197..c762436 100644 --- a/inserter/Cargo.lock +++ b/inserter/Cargo.lock @@ -210,7 +210,7 @@ dependencies = [ [[package]] name = "dsp_seed_finder" -version = "0.1.0" +version = "0.7.0" dependencies = [ "anyhow", "crossbeam-channel", diff --git a/inserter/Cargo.toml b/inserter/Cargo.toml index 89ab8e0..537b127 100644 --- a/inserter/Cargo.toml +++ b/inserter/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dsp_seed_finder" -version = "0.1.0" +version = "0.7.0" edition = "2021" [profile.release] @@ -24,4 +24,3 @@ crossterm = "0.29.0" anyhow = "1.0.103" lazy_static = "1.5.0" itertools = "0.15.0" - diff --git a/inserter/src/algorithm/data/planet.rs b/inserter/src/algorithm/data/planet.rs index 625a1f0..dbebec7 100644 --- a/inserter/src/algorithm/data/planet.rs +++ b/inserter/src/algorithm/data/planet.rs @@ -2,8 +2,6 @@ use crate::algorithm::data::birth_points::BirthPoints; use crate::algorithm::data::game_desc::GameDesc; use crate::algorithm::data::planet_raw_data::PlanetRawData; use crate::algorithm::data::vector_f2::VectorF2; -use std::convert::TryFrom; - use super::enums::{OceanType, PlanetType, SpectrType, StarType, ThemeDistribute, VeinType}; use super::pose::Pose; use super::quaternion::Quaternion; @@ -989,15 +987,19 @@ impl<'a> Planet<'a> { normal_dir += birth_point; } normal_dir.normalize(); - if self.can_place_vein(algo_id, &vein_type, &normal_dir, &mut raw_data) { - let not_too_close_to_other_vein = - vein_vectors.iter().all(|(_, pos, _)| { - (pos.distance_sq_from(&normal_dir) as f64) >= min_sq_dist - }); - if not_too_close_to_other_vein { - vein_vectors.push((vein_type, normal_dir, false)); - break; - } + let not_too_close_to_other_vein = vein_vectors.iter().all(|(_, pos, _)| { + (pos.distance_sq_from(&normal_dir) as f64) >= min_sq_dist + }); + if not_too_close_to_other_vein + && self.can_place_vein( + algo_id, + &vein_type, + &normal_dir, + &mut raw_data, + ) + { + vein_vectors.push((vein_type, normal_dir, false)); + break; } } if vein_vectors.len() >= 512 { @@ -1185,4 +1187,4 @@ const SEGMENT_TABLE: [i32; 512] = [ 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, -]; \ No newline at end of file +]; diff --git a/inserter/src/algorithm/data/planet_algorithms/algo5.rs b/inserter/src/algorithm/data/planet_algorithms/algo5.rs index 524a9d4..595e2b6 100644 --- a/inserter/src/algorithm/data/planet_algorithms/algo5.rs +++ b/inserter/src/algorithm/data/planet_algorithms/algo5.rs @@ -87,13 +87,12 @@ impl PlanetAlgorithm for PlanetAlgorithm5 { 0.0 }; - let detail_noise = self - .noise2 - .noise_3d_fbm(xin * 1.5, yin * 1.5, zin * 1.5, 2, 0.5, 2.0) - .abs(); - let mut terrain_height = height_base - crack_intensity * 1.2 * fluid_clamped; if terrain_height >= 0.0 { + let detail_noise = self + .noise2 + .noise_3d_fbm(xin * 1.5, yin * 1.5, zin * 1.5, 2, 0.5, 2.0) + .abs(); terrain_height += cell_noise * 0.25 + detail_noise * 0.6; } diff --git a/inserter/src/algorithm/data/planet_algorithms/algo6.rs b/inserter/src/algorithm/data/planet_algorithms/algo6.rs index 7aaf9eb..eacd2f3 100644 --- a/inserter/src/algorithm/data/planet_algorithms/algo6.rs +++ b/inserter/src/algorithm/data/planet_algorithms/algo6.rs @@ -87,13 +87,12 @@ impl PlanetAlgorithm for PlanetAlgorithm6 { 0.0 }; - let detail_noise = self - .noise2 - .noise_3d_fbm(xin * 1.5, yin * 1.5, zin * 1.5, 2, 0.5, 2.0) - .abs(); - let mut terrain_height = height_base - crack_intensity * 1.2 * fluid_clamped; if terrain_height >= 0.0 { + let detail_noise = self + .noise2 + .noise_3d_fbm(xin * 1.5, yin * 1.5, zin * 1.5, 2, 0.5, 2.0) + .abs(); terrain_height += cell_noise * 0.25 + detail_noise * 0.6; } diff --git a/inserter/src/algorithm/data/planet_algorithms/algo9.rs b/inserter/src/algorithm/data/planet_algorithms/algo9.rs index 262758c..49dbad4 100644 --- a/inserter/src/algorithm/data/planet_algorithms/algo9.rs +++ b/inserter/src/algorithm/data/planet_algorithms/algo9.rs @@ -99,21 +99,7 @@ impl PlanetAlgorithm for PlanetAlgorithm9 { 2.0, ) * noise_amplitude + noise_offset; - let layer4_noise = self.noise2.noise_3d_fbm( - world_x * (1.0 / 400.0), - world_y * (1.0 / 400.0), - world_z * (1.0 / 400.0), - 3, - 0.5, - 2.0, - ) * noise_amplitude - * noise2_amplitude - + noise2_offset; - let clamped_layer4 = if layer4_noise > 0.0 { - layer4_noise * 0.5 - } else { - layer4_noise - }; + let clamped_layer4 = clamped_layer2; let alt_height = ((layer3_noise + clamped_layer4 + 5.0) * 0.13).powf(6.0) * 24.0 - 24.0; diff --git a/inserter/src/algorithm/data/planet_raw_data.rs b/inserter/src/algorithm/data/planet_raw_data.rs index 9f69138..be00a77 100644 --- a/inserter/src/algorithm/data/planet_raw_data.rs +++ b/inserter/src/algorithm/data/planet_raw_data.rs @@ -5,33 +5,66 @@ use crate::algorithm::data::planet_grid::{ }; use super::vector_f3::VectorF3; +use std::cell::RefCell; use std::f64::consts::PI; +const VALID_WORDS: usize = DATA_LENGTH.div_ceil(u64::BITS as usize); + +#[derive(Default)] +struct TerrainCache { + heights: Vec, + valid: Vec, + touched: Vec, +} + +impl TerrainCache { + fn new() -> Self { + Self { + heights: vec![0; DATA_LENGTH], + valid: vec![0; VALID_WORDS], + touched: Vec::new(), + } + } + + fn clear(&mut self) { + for index in self.touched.drain(..) { + self.valid[index / u64::BITS as usize] &= !(1 << (index % u64::BITS as usize)); + } + } +} + +thread_local! { + static TERRAIN_CACHE_POOL: RefCell> = const { RefCell::new(Vec::new()) }; +} + pub struct PlanetRawData { grid: &'static PlanetGrid, algo: Box, - cache: Vec, + cache: TerrainCache, } impl PlanetRawData { pub fn new(planet: &Planet) -> Self { + let cache = TERRAIN_CACHE_POOL + .with(|pool| pool.borrow_mut().pop()) + .unwrap_or_else(TerrainCache::new); Self { grid: get_planet_grid(), algo: create_and_prepare_algo(planet), - cache: vec![f32::NAN; DATA_LENGTH], + cache, } } #[inline] fn get_height(&mut self, index: usize) -> f32 { - let val = self.cache[index]; - if val.is_nan() { - let h = (self.algo.get_height(index) * 100.0) as u16 as f32; - self.cache[index] = h; - h - } else { - val + let word = index / u64::BITS as usize; + let mask = 1 << (index % u64::BITS as usize); + if self.cache.valid[word] & mask == 0 { + self.cache.heights[index] = (self.algo.get_height(index) * 100.0) as u16; + self.cache.valid[word] |= mask; + self.cache.touched.push(index); } + self.cache.heights[index] as f32 } pub fn query_height_normalized(&mut self, vpos_normalized: &VectorF3) -> f32 { @@ -73,3 +106,11 @@ impl PlanetRawData { self.query_height_normalized(&vpos) } } + +impl Drop for PlanetRawData { + fn drop(&mut self) { + self.cache.clear(); + let cache = std::mem::take(&mut self.cache); + TERRAIN_CACHE_POOL.with(|pool| pool.borrow_mut().push(cache)); + } +} diff --git a/inserter/src/algorithm/data/simplex_noise.rs b/inserter/src/algorithm/data/simplex_noise.rs index 23fc007..54eb775 100644 --- a/inserter/src/algorithm/data/simplex_noise.rs +++ b/inserter/src/algorithm/data/simplex_noise.rs @@ -25,8 +25,8 @@ const G3: f64 = 1.0 / 6.0; /// SimplexNoise implementation with DSP-specific permutation. pub struct SimplexNoise { - perm: [i16; 512], - perm_mod12: [i16; 512], + perm: [u8; 512], + perm_mod12: [u8; 512], } impl SimplexNoise { @@ -42,7 +42,7 @@ impl SimplexNoise { /// Initialize with seed-shuffled permutation (using DspRandom). fn init_member_seed(&mut self, seed: i32) { - let mut p: [i16; 256] = std::array::from_fn(|i| i as i16); + let mut p: [u8; 256] = std::array::from_fn(|i| i as u8); // Only shuffle when seed != 0 (matching C# behavior where seed 0 might be special) // Actually the C# always shuffles. Let's shuffle based on seed. @@ -54,10 +54,12 @@ impl SimplexNoise { p[i2] = num; } - for i in 0..512 { - self.perm[i] = p[i & 255]; + for (i, value) in p.into_iter().enumerate() { + self.perm[i] = value; self.perm_mod12[i] = self.perm[i] % 12; } + self.perm.copy_within(..256, 256); + self.perm_mod12.copy_within(..256, 256); } /// 3D simplex noise. diff --git a/inserter/src/bin/seed_output.rs b/inserter/src/bin/seed_output.rs new file mode 100644 index 0000000..237f647 --- /dev/null +++ b/inserter/src/bin/seed_output.rs @@ -0,0 +1,57 @@ +#[path = "../algorithm/mod.rs"] +mod algorithm; +#[path = "../generate_csv.rs"] +mod generate_csv; + +use anyhow::{bail, Context, Result}; +use std::env; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::Path; + +const STAR_COUNT: usize = 64; +const REC_MULTIPLIER: f32 = 1.0; +const OUTPUT_MAGIC: &[u8] = b"DSPSEED1"; + +fn parse_seed(value: Option, name: &str) -> Result { + value + .with_context(|| format!("missing {name}"))? + .parse() + .with_context(|| format!("invalid {name}")) +} + +fn write_record(writer: &mut impl Write, seed: i32, data: &[u8]) -> Result<()> { + writer.write_all(&seed.to_be_bytes())?; + writer.write_all(&(data.len() as u64).to_be_bytes())?; + writer.write_all(data)?; + Ok(()) +} + +fn run() -> Result<()> { + let mut args = env::args().skip(1); + let start_seed = parse_seed(args.next(), "start seed")?; + let end_seed = parse_seed(args.next(), "end seed")?; + let output = args.next().context("missing output path")?; + if args.next().is_some() { + bail!("too many arguments"); + } + if start_seed >= end_seed { + bail!("start seed must be less than end seed"); + } + + let file = File::create(Path::new(&output)) + .with_context(|| format!("cannot create output file: {output}"))?; + let mut writer = BufWriter::new(file); + writer.write_all(OUTPUT_MAGIC)?; + for seed in start_seed..end_seed { + let (stars, planets) = generate_csv::gen_formatted(seed)?; + write_record(&mut writer, seed, &stars)?; + write_record(&mut writer, seed, &planets)?; + } + writer.flush()?; + Ok(()) +} + +fn main() -> Result<()> { + run() +} diff --git a/inserter/src/generate_csv.rs b/inserter/src/generate_csv.rs index e4f587c..2cdfd4e 100644 --- a/inserter/src/generate_csv.rs +++ b/inserter/src/generate_csv.rs @@ -3,23 +3,33 @@ use crate::algorithm::data::enums::{PlanetType, ORES}; use crate::algorithm::data::game_desc::GameDesc; use crate::algorithm::generate_stars; -const COPY_HEADER: &[u8] = b"PGCOPY\n\xFF\r\n\0\0"; -const COPY_FOOTER: &[u8] = b"\xFF\xFF"; +pub const COPY_SIGNATURE: &[u8] = b"PGCOPY\n\xFF\r\n\0"; +pub const COPY_HEADER_FIELDS: &[u8] = &[0; 8]; +pub const COPY_FOOTER: &[u8] = b"\xFF\xFF"; + +#[inline] +fn write_row_header(buf: &mut Vec, columns: i16) { + buf.extend_from_slice(&columns.to_be_bytes()); +} #[inline] fn write_i32(buf: &mut Vec, v: i32) { + buf.extend_from_slice(&4i32.to_be_bytes()); buf.extend_from_slice(&v.to_be_bytes()); } #[inline] fn write_i16(buf: &mut Vec, v: i16) { + buf.extend_from_slice(&2i32.to_be_bytes()); buf.extend_from_slice(&v.to_be_bytes()); } #[inline] fn write_f32(buf: &mut Vec, v: f32) { + buf.extend_from_slice(&4i32.to_be_bytes()); buf.extend_from_slice(&v.to_be_bytes()); } #[inline] fn write_bool(buf: &mut Vec, v: bool) { + buf.extend_from_slice(&1i32.to_be_bytes()); buf.push(v as u8); } #[inline] @@ -35,25 +45,16 @@ pub fn gen_formatted(seed: i32) -> anyhow::Result<(Vec, Vec)> { let hab_count = std::cell::Cell::new(0i32); let galaxy = generate_stars(seed, &game_desc, &hab_count); - let mut stars: Vec = Vec::with_capacity(crate::STAR_COUNT * 96); + let mut stars: Vec = Vec::with_capacity(crate::STAR_COUNT * 176); let mut planets: Vec = Vec::with_capacity(crate::STAR_COUNT * 256 * 5); - stars.extend_from_slice(COPY_HEADER); - // flags (4 bytes) + header extension length (4 bytes) - stars.extend_from_slice(&0i32.to_be_bytes()); - stars.extend_from_slice(&0i32.to_be_bytes()); - - planets.extend_from_slice(COPY_HEADER); - planets.extend_from_slice(&0i32.to_be_bytes()); - planets.extend_from_slice(&0i32.to_be_bytes()); - for solar_system in galaxy { let star = solar_system.star.clone(); let star_id = star.index as i32 + seed * 100; // Stars: 22 columns // Columns: id, seed, start_dist, star_index, luminosity, dyson_radius, type, spectr, ore_iron..ore_mag - write_i16(&mut stars, 22); + write_row_header(&mut stars, 22); write_i32(&mut stars, star_id); write_i32(&mut stars, seed); write_f32(&mut stars, star.position.magnitude() as f32); @@ -105,13 +106,17 @@ pub fn gen_formatted(seed: i32) -> anyhow::Result<(Vec, Vec)> { // Columns: star_id, index, orbiting, gas_giant, sun_distance, inside_ds, // satellites, theme_id, gas_h, gas_d, gas_i, tidal_lock, // ore_iron..ore_mag - write_i16(&mut planets, 26); + let sun_distance = planet.get_sun_distance(); + write_row_header(&mut planets, 26); write_i32(&mut planets, star_id); write_i16(&mut planets, planet.index as i16); write_i16(&mut planets, orbiting); write_bool(&mut planets, planet.get_type() == &PlanetType::Gas); - write_f32(&mut planets, planet.get_orbital_radius()); - write_bool(&mut planets, planet.get_orbital_radius() * 40000.0 < star.get_dyson_radius() as f32); + write_f32(&mut planets, sun_distance); + write_bool( + &mut planets, + sun_distance * 40000.0 < star.get_dyson_radius() as f32, + ); write_i16(&mut planets, satellite_count); write_i16(&mut planets, planet.get_theme().id as i16); write_f32(&mut planets, gas_h); @@ -141,8 +146,35 @@ pub fn gen_formatted(seed: i32) -> anyhow::Result<(Vec, Vec)> { } } - stars.extend_from_slice(COPY_FOOTER); - planets.extend_from_slice(COPY_FOOTER); - Ok((stars, planets)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn binary_fields_include_lengths() { + let mut output = Vec::new(); + write_i16(&mut output, 7); + write_i32(&mut output, 11); + write_f32(&mut output, 1.5); + write_bool(&mut output, true); + write_null_col(&mut output); + + assert_eq!( + output, + [ + 0, 0, 0, 2, 0, 7, 0, 0, 0, 4, 0, 0, 0, 11, 0, 0, 0, 4, 63, 192, 0, 0, 0, + 0, 0, 1, 1, 255, 255, 255, 255, + ] + ); + } + + #[test] + fn binary_copy_framing_has_protocol_lengths() { + assert_eq!(COPY_SIGNATURE.len(), 11); + assert_eq!(COPY_HEADER_FIELDS.len(), 8); + assert_eq!(COPY_FOOTER, [255, 255]); + } +} diff --git a/inserter/src/main.rs b/inserter/src/main.rs index ed771aa..067b06b 100644 --- a/inserter/src/main.rs +++ b/inserter/src/main.rs @@ -42,10 +42,13 @@ static PROGRESS_WORKERS: [AtomicI32; MAX_WORKERS] = [const { AtomicI32::new(0) } lazy_static! { static ref START_SEED: i32 = env_int!("START_SEED", 0); static ref END_SEED: i32 = env_int!("END_SEED", 10_000); - static ref WORKER_THREADS: i32 = env_int!("WORKER_THREADS", 8); - static ref WRITER_THREADS: i32 = env_int!("WRITER_THREADS", 4); - static ref COMMIT_COUNT: usize = env_int!("COMMIT_COUNT", 1000) as usize; - static ref CHANNEL_SIZE: usize = env_int!("CHANNEL_SIZE", 1000) as usize; + static ref WORKER_THREADS: i32 = std::env::var("WORKER_THREADS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or_else(|| (*END_SEED - *START_SEED).clamp(1, MAX_WORKERS as i32)); + static ref WRITER_THREADS: i32 = env_int!("WRITER_THREADS", 1); + static ref COMMIT_COUNT: usize = env_int!("COMMIT_COUNT", 64) as usize; + static ref CHANNEL_SIZE: usize = env_int!("CHANNEL_SIZE", 64) as usize; static ref CHECKPOINT_FILE: String = env_str!("CHECKPOINT_FILE", "checkpoints.txt"); static ref OVERRIDE_CHECKPOINTS: bool = env_bool!("OVERRIDE_CHECKPOINTS"); @@ -61,7 +64,11 @@ lazy_static! { static ref PG_DBNAME: String = env_str!("PG_DBNAME", "dsp"); static ref CP_INTERVAL: Option = get_env_interval("CHECKPOINT_INTERVAL"); - static ref LOG_INTERVAL: Option = get_env_interval("LOG_INTERVAL"); + static ref LOG_INTERVAL: Option = if std::env::var_os("LOG_INTERVAL").is_some() { + get_env_interval("LOG_INTERVAL") + } else { + None + }; static ref DB_STR: String = { @@ -74,7 +81,6 @@ lazy_static! { PG_DBNAME.as_str() ) }; - static ref MAX_BUFFER: usize = *CHANNEL_SIZE + *COMMIT_COUNT * *WORKER_THREADS as usize - 2; static ref LOG_NAME: String = { let num = std::fs::read_dir(&*LOG_DIR).map(|x| { x.map(|x| x.unwrap().file_name().into_string().unwrap()) diff --git a/inserter/src/misc.rs b/inserter/src/misc.rs index a741d5d..82c2a40 100644 --- a/inserter/src/misc.rs +++ b/inserter/src/misc.rs @@ -95,8 +95,11 @@ pub fn create_db_schema() -> bool { pub fn check_db_connection() -> bool { crate::log_info!( - "Checking database connection to {}", - *crate::DB_STR + "Checking PostgreSQL connection to {}:{}/{} as {}", + *crate::PG_NETLOC, + *crate::PG_PORT, + *crate::PG_DBNAME, + *crate::PG_USER, ); match postgres::Client::connect((*crate::DB_STR).as_str(), postgres::NoTls) { @@ -190,6 +193,7 @@ pub fn validate_config() -> anyhow::Result<()> { } pub fn write_log() -> anyhow::Result<()> { + std::fs::create_dir_all(&*crate::LOG_DIR)?; let log_path = Path::new(&*crate::LOG_DIR).join(&*crate::LOG_NAME); if !log_path.exists() { @@ -213,4 +217,4 @@ macro_rules! process_option { ($user_option:expr) => { $user_option.to_ascii_lowercase().replace(" ", "").replace("_", "").replace("-", "") }; -} \ No newline at end of file +} diff --git a/inserter/src/threads.rs b/inserter/src/threads.rs index 65d0530..202fe83 100644 --- a/inserter/src/threads.rs +++ b/inserter/src/threads.rs @@ -5,7 +5,9 @@ use std::io::Write; use crossbeam_channel::{Receiver, RecvTimeoutError, Sender}; use postgres::{Client, NoTls}; use anyhow::Result; -use crate::generate_csv::gen_formatted; +use crate::generate_csv::{ + gen_formatted, COPY_FOOTER, COPY_HEADER_FIELDS, COPY_SIGNATURE, +}; use crate::{log_info, COMMITTED_SEEDS, COMMIT_COUNT, DB_STR, PROGRESS_WORKERS}; use crate::misc::{COPY_PLANET, COPY_STAR}; @@ -21,14 +23,24 @@ pub fn worker_thread(seeds: Range, send: Sender<(Vec, Vec)>, id: us } pub fn commit_thread(rec: Receiver<(Vec, Vec)>) -> Result<()> { let mut client = Client::connect(&*DB_STR.as_str(), NoTls)?; + let mut batch: Vec<(Vec, Vec)> = Vec::with_capacity(*COMMIT_COUNT); - 'outer: loop { - let mut batch: Vec<(Vec, Vec)> = Vec::with_capacity(*COMMIT_COUNT); - 'inner: for i in 0..*COMMIT_COUNT { + loop { + batch.clear(); + match rec.recv() { + Ok(msg) => batch.push(msg), + Err(_) => break, + } + + let mut disconnected = false; + for _ in 1..*COMMIT_COUNT { match rec.recv_timeout(Duration::new(1, 0)) { Ok(msg) => batch.push(msg), - Err(RecvTimeoutError::Timeout) => panic!("commit_thread: recv_timeout reached - channel stall detected (>1s lull)"), - Err(RecvTimeoutError::Disconnected) => if i == 0 {break 'outer} else {break 'inner}, + Err(RecvTimeoutError::Timeout) => break, + Err(RecvTimeoutError::Disconnected) => { + disconnected = true; + break; + } } } @@ -36,33 +48,36 @@ pub fn commit_thread(rec: Receiver<(Vec, Vec)>) -> Result<()> { { let mut scpy = txn.copy_in(COPY_STAR)?; + scpy.write_all(COPY_SIGNATURE)?; + scpy.write_all(COPY_HEADER_FIELDS)?; for (star, _) in &batch { - scpy.write_all(star).expect("writing to scpy failed"); + scpy.write_all(star)?; } + scpy.write_all(COPY_FOOTER)?; scpy.finish()?; } { let mut pcpy = txn.copy_in(COPY_PLANET)?; + pcpy.write_all(COPY_SIGNATURE)?; + pcpy.write_all(COPY_HEADER_FIELDS)?; for (_, planet) in &batch { - pcpy.write_all(planet).expect("writing to pcpy failed"); + pcpy.write_all(planet)?; } + pcpy.write_all(COPY_FOOTER)?; pcpy.finish()?; } txn.commit()?; - COMMITTED_SEEDS.fetch_add(batch.len() as i32, std::sync::atomic::Ordering::SeqCst); + COMMITTED_SEEDS.fetch_add(batch.len() as i32, Relaxed); + if disconnected { + break; + } } log_info!("Writer thread terminated"); Ok(()) } pub fn writer_sink(rec: Receiver<(Vec, Vec)>) -> Result<()> { - loop { - match rec.recv_timeout(Duration::new(1, 0)) { - Ok(_) => {}, - Err(RecvTimeoutError::Timeout) => panic!("writer_sink: recv_timeout reached - channel stall detected (>1s lull)"), - Err(RecvTimeoutError::Disconnected) => break, - } - } + while rec.recv().is_ok() {} Ok(()) -} \ No newline at end of file +}