From 5d45b5ad287bf2996cc9110254184f027b12d6cd Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 16:09:12 +0000 Subject: [PATCH 01/18] Add reproducible benchmark tools Add an A-B-B-A benchmark runner that records raw throughput, CPU, memory, page-fault, and context-switch measurements in JSON. Use robust median, MAD, percentile, and bootstrap statistics so host noise is visible. Add a canonical seed-output utility that writes records in seed order. This gives optimization work a byte-for-byte determinism gate without changing the production executable. The initial 0..100 output digest is 3f26eb1b456d5a7b71f40a98ab03e0472b05e41cc31e65740dca002da6afa7d6. This commit does not change production throughput or generated bytes. --- benchmarks/run_comparison.py | 231 ++++++++++++++++++++++++++++++++ inserter/src/bin/seed_output.rs | 57 ++++++++ 2 files changed, 288 insertions(+) create mode 100755 benchmarks/run_comparison.py create mode 100644 inserter/src/bin/seed_output.rs diff --git a/benchmarks/run_comparison.py b/benchmarks/run_comparison.py new file mode 100755 index 0000000..f3c8ddd --- /dev/null +++ b/benchmarks/run_comparison.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Compare two inserter executables with controlled process runs.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import random +import re +import statistics +import subprocess +import tempfile +from datetime import datetime, timezone +from pathlib import Path + + +THROUGHPUT_PATTERN = re.compile(r"^seeds/sec: ([0-9.]+)$", re.MULTILINE) +TIME_FIELDS = ( + "wall_seconds", + "user_seconds", + "system_seconds", + "max_rss_kib", + "minor_page_faults", + "major_page_faults", + "voluntary_context_switches", + "involuntary_context_switches", +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--candidate", required=True, type=Path) + parser.add_argument("--candidate-name", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--start-seed", type=int, default=0) + parser.add_argument("--end-seed", type=int, default=60) + parser.add_argument("--workers", type=int, default=1) + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--cpus", default="0") + return parser.parse_args() + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def percentile(values: list[float], fraction: float) -> float: + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + weight = position - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def describe(values: list[float]) -> dict[str, float]: + median = statistics.median(values) + deviations = [abs(value - median) for value in values] + return { + "median": median, + "mad": statistics.median(deviations), + "p25": percentile(values, 0.25), + "p75": percentile(values, 0.75), + "minimum": min(values), + "maximum": max(values), + } + + +def bootstrap_ci(values: list[float], samples: int = 20_000) -> list[float]: + generator = random.Random(0) + medians = [] + for _ in range(samples): + draw = [generator.choice(values) for _ in values] + medians.append(statistics.median(draw)) + return [percentile(medians, 0.025), percentile(medians, 0.975)] + + +def run_once( + label: str, + executable: Path, + args: argparse.Namespace, + sequence: int, +) -> dict[str, object]: + environment = os.environ.copy() + environment.update( + { + "BENCHMARK": "1", + "START_SEED": str(args.start_seed), + "END_SEED": str(args.end_seed), + "WORKER_THREADS": str(args.workers), + "WRITER_THREADS": "1", + "CHANNEL_SIZE": "64", + "COMMIT_COUNT": "64", + "NO_TUI": "1", + "LOG_LEVEL": "error", + "LOG_INTERVAL": "none", + "CHECKPOINT_INTERVAL": "none", + "OVERRIDE_CHECKPOINTS": "1", + "CHECKPOINT_FILE": "/tmp/dsp-benchmark-no-checkpoint", + "PG_NETLOC": "127.0.0.1", + "PG_PORT": "1", + } + ) + + with tempfile.NamedTemporaryFile() as time_output: + command = [ + "/usr/bin/time", + "-f", + "%e\n%U\n%S\n%M\n%R\n%F\n%w\n%c", + "-o", + time_output.name, + "taskset", + "-c", + args.cpus, + str(executable), + ] + completed = subprocess.run( + command, + env=environment, + check=False, + text=True, + capture_output=True, + ) + time_output.seek(0) + measurements = time_output.read().decode("ascii").splitlines() + + if completed.returncode != 0: + raise RuntimeError( + f"{label} failed with status {completed.returncode}:\n{completed.stderr}" + ) + match = THROUGHPUT_PATTERN.search(completed.stdout) + if match is None or len(measurements) != len(TIME_FIELDS): + raise RuntimeError(f"Cannot parse {label} output:\n{completed.stdout}") + + values = dict(zip(TIME_FIELDS, measurements, strict=True)) + integer_fields = set(TIME_FIELDS[3:]) + result: dict[str, object] = { + "sequence": sequence, + "variant": label, + "throughput_seeds_per_second": float(match.group(1)), + } + for field, value in values.items(): + result[field] = int(value) if field in integer_fields else float(value) + return result + + +def summarize(results: list[dict[str, object]], candidate_name: str) -> dict[str, object]: + by_variant = { + label: [ + float(result["throughput_seeds_per_second"]) + for result in results + if result["variant"] == label + ] + for label in ("baseline", candidate_name) + } + baseline = by_variant["baseline"] + candidate = by_variant[candidate_name] + paired_changes = [ + (new / old - 1.0) * 100.0 for old, new in zip(baseline, candidate, strict=True) + ] + return { + "throughput": { + "baseline": describe(baseline), + candidate_name: describe(candidate), + }, + "paired_change_percent": { + **describe(paired_changes), + "bootstrap_95_percent_ci": bootstrap_ci(paired_changes), + }, + } + + +def main() -> None: + args = parse_args() + for executable in (args.baseline, args.candidate): + if not executable.is_file(): + raise SystemExit(f"Executable does not exist: {executable}") + + variants = { + "baseline": args.baseline.resolve(), + args.candidate_name: args.candidate.resolve(), + } + order = ["baseline", args.candidate_name, args.candidate_name, "baseline"] + results = [] + for _ in range(args.rounds): + for label in order: + result = run_once(label, variants[label], args, len(results)) + results.append(result) + print( + f"{label}: {result['throughput_seeds_per_second']:.6f} seeds/s", + flush=True, + ) + + artifact = { + "schema_version": 1, + "created_utc": datetime.now(timezone.utc).isoformat(), + "host": { + "platform": platform.platform(), + "processor_count": os.cpu_count(), + "cpu_set": args.cpus, + }, + "workload": { + "start_seed": args.start_seed, + "end_seed": args.end_seed, + "seed_count": args.end_seed - args.start_seed, + "workers": args.workers, + "writer_sinks": 1, + "channel_size": 64, + }, + "executables": { + label: {"path": str(path), "sha256": file_sha256(path)} + for label, path in variants.items() + }, + "results": results, + "summary": summarize(results, args.candidate_name), + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(artifact, indent=2) + "\n", encoding="ascii") + print(json.dumps(artifact["summary"], indent=2)) + + +if __name__ == "__main__": + main() diff --git a/inserter/src/bin/seed_output.rs b/inserter/src/bin/seed_output.rs new file mode 100644 index 0000000..237f647 --- /dev/null +++ b/inserter/src/bin/seed_output.rs @@ -0,0 +1,57 @@ +#[path = "../algorithm/mod.rs"] +mod algorithm; +#[path = "../generate_csv.rs"] +mod generate_csv; + +use anyhow::{bail, Context, Result}; +use std::env; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::Path; + +const STAR_COUNT: usize = 64; +const REC_MULTIPLIER: f32 = 1.0; +const OUTPUT_MAGIC: &[u8] = b"DSPSEED1"; + +fn parse_seed(value: Option, name: &str) -> Result { + value + .with_context(|| format!("missing {name}"))? + .parse() + .with_context(|| format!("invalid {name}")) +} + +fn write_record(writer: &mut impl Write, seed: i32, data: &[u8]) -> Result<()> { + writer.write_all(&seed.to_be_bytes())?; + writer.write_all(&(data.len() as u64).to_be_bytes())?; + writer.write_all(data)?; + Ok(()) +} + +fn run() -> Result<()> { + let mut args = env::args().skip(1); + let start_seed = parse_seed(args.next(), "start seed")?; + let end_seed = parse_seed(args.next(), "end seed")?; + let output = args.next().context("missing output path")?; + if args.next().is_some() { + bail!("too many arguments"); + } + if start_seed >= end_seed { + bail!("start seed must be less than end seed"); + } + + let file = File::create(Path::new(&output)) + .with_context(|| format!("cannot create output file: {output}"))?; + let mut writer = BufWriter::new(file); + writer.write_all(OUTPUT_MAGIC)?; + for seed in start_seed..end_seed { + let (stars, planets) = generate_csv::gen_formatted(seed)?; + write_record(&mut writer, seed, &stars)?; + write_record(&mut writer, seed, &planets)?; + } + writer.flush()?; + Ok(()) +} + +fn main() -> Result<()> { + run() +} From cba9ac9f810097211a1162825b177c2927ba3851 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 16:18:23 +0000 Subject: [PATCH 02/18] Reuse compact terrain caches Store quantized terrain heights as u16 values with a validity bit set. Retain one cache pool per worker thread and clear only the entries that the prior planet used. This removes the 646 KiB allocation and full NaN initialization for each solid planet. The implementation uses safe Rust and preserves the prior u16-to-f32 value conversion. For seeds 0 through 59 on one pinned CPU, median throughput increased from 2.718 to 3.455 seeds per second. The paired median gain was 27.14%, with a bootstrap 95% interval from 16.58% to 31.35%. In a 100-seed output run, minor page faults fell from 3,319,980 to 7,619 and system CPU time fell from 6.44 seconds to 0.17 seconds. The canonical 0 through 99 output remained byte-for-byte equal at SHA-256 3f26eb1b456d5a7b71f40a98ab03e0472b05e41cc31e65740dca002da6afa7d6. --- .../src/algorithm/data/planet_raw_data.rs | 59 ++++++++++++++++--- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/inserter/src/algorithm/data/planet_raw_data.rs b/inserter/src/algorithm/data/planet_raw_data.rs index 9f69138..be00a77 100644 --- a/inserter/src/algorithm/data/planet_raw_data.rs +++ b/inserter/src/algorithm/data/planet_raw_data.rs @@ -5,33 +5,66 @@ use crate::algorithm::data::planet_grid::{ }; use super::vector_f3::VectorF3; +use std::cell::RefCell; use std::f64::consts::PI; +const VALID_WORDS: usize = DATA_LENGTH.div_ceil(u64::BITS as usize); + +#[derive(Default)] +struct TerrainCache { + heights: Vec, + valid: Vec, + touched: Vec, +} + +impl TerrainCache { + fn new() -> Self { + Self { + heights: vec![0; DATA_LENGTH], + valid: vec![0; VALID_WORDS], + touched: Vec::new(), + } + } + + fn clear(&mut self) { + for index in self.touched.drain(..) { + self.valid[index / u64::BITS as usize] &= !(1 << (index % u64::BITS as usize)); + } + } +} + +thread_local! { + static TERRAIN_CACHE_POOL: RefCell> = const { RefCell::new(Vec::new()) }; +} + pub struct PlanetRawData { grid: &'static PlanetGrid, algo: Box, - cache: Vec, + cache: TerrainCache, } impl PlanetRawData { pub fn new(planet: &Planet) -> Self { + let cache = TERRAIN_CACHE_POOL + .with(|pool| pool.borrow_mut().pop()) + .unwrap_or_else(TerrainCache::new); Self { grid: get_planet_grid(), algo: create_and_prepare_algo(planet), - cache: vec![f32::NAN; DATA_LENGTH], + cache, } } #[inline] fn get_height(&mut self, index: usize) -> f32 { - let val = self.cache[index]; - if val.is_nan() { - let h = (self.algo.get_height(index) * 100.0) as u16 as f32; - self.cache[index] = h; - h - } else { - val + let word = index / u64::BITS as usize; + let mask = 1 << (index % u64::BITS as usize); + if self.cache.valid[word] & mask == 0 { + self.cache.heights[index] = (self.algo.get_height(index) * 100.0) as u16; + self.cache.valid[word] |= mask; + self.cache.touched.push(index); } + self.cache.heights[index] as f32 } pub fn query_height_normalized(&mut self, vpos_normalized: &VectorF3) -> f32 { @@ -73,3 +106,11 @@ impl PlanetRawData { self.query_height_normalized(&vpos) } } + +impl Drop for PlanetRawData { + fn drop(&mut self) { + self.cache.clear(); + let cache = std::mem::take(&mut self.cache); + TERRAIN_CACHE_POOL.with(|pool| pool.borrow_mut().push(cache)); + } +} From 202901b822f3a8caa0fa1ec649dee783b3735481 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 16:29:06 +0000 Subject: [PATCH 03/18] Reject close veins before terrain Check vein-group spacing before the terrain placement rule. Distance checks are substantially cheaper than terrain interpolation and simplex-noise generation. The two predicates use the same candidate, consume no random values, and have the same accepted set, so this changes only avoided work. For seeds 0 through 99 on one pinned CPU, median throughput increased from 3.438 to 3.620 seeds per second after the terrain-cache optimization. The paired median gain was 5.14%, with a bootstrap 95% interval from 2.62% to 8.39%. The canonical 0 through 99 output remained byte-for-byte equal at SHA-256 3f26eb1b456d5a7b71f40a98ab03e0472b05e41cc31e65740dca002da6afa7d6. --- inserter/src/algorithm/data/planet.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/inserter/src/algorithm/data/planet.rs b/inserter/src/algorithm/data/planet.rs index 625a1f0..903e23a 100644 --- a/inserter/src/algorithm/data/planet.rs +++ b/inserter/src/algorithm/data/planet.rs @@ -989,15 +989,19 @@ impl<'a> Planet<'a> { normal_dir += birth_point; } normal_dir.normalize(); - if self.can_place_vein(algo_id, &vein_type, &normal_dir, &mut raw_data) { - let not_too_close_to_other_vein = - vein_vectors.iter().all(|(_, pos, _)| { - (pos.distance_sq_from(&normal_dir) as f64) >= min_sq_dist - }); - if not_too_close_to_other_vein { - vein_vectors.push((vein_type, normal_dir, false)); - break; - } + let not_too_close_to_other_vein = vein_vectors.iter().all(|(_, pos, _)| { + (pos.distance_sq_from(&normal_dir) as f64) >= min_sq_dist + }); + if not_too_close_to_other_vein + && self.can_place_vein( + algo_id, + &vein_type, + &normal_dir, + &mut raw_data, + ) + { + vein_vectors.push((vein_type, normal_dir, false)); + break; } } if vein_vectors.len() >= 512 { @@ -1185,4 +1189,4 @@ const SEGMENT_TABLE: [i32; 512] = [ 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, 500, -]; \ No newline at end of file +]; From cbbb97c15b27ff6cba1a04d78d344f461509b8ab Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 17:11:20 +0000 Subject: [PATCH 04/18] Compact simplex permutation tables Store simplex permutation and modulo tables as u8 values. Their ranges are 0 through 255 and 0 through 11. Initialize the first 256 entries and copy them into the duplicate second half. This halves each simplex table footprint and removes redundant modulo work without changing random calls or numeric operations. For seeds 0 through 99 on one pinned CPU, median throughput increased from 3.529 to 3.646 seeds per second after the prior optimizations. The paired median gain was 2.95%, with a bootstrap 95% interval from 0.91% to 7.56%. The canonical 0 through 99 output remained byte-for-byte equal at SHA-256 3f26eb1b456d5a7b71f40a98ab03e0472b05e41cc31e65740dca002da6afa7d6. --- inserter/src/algorithm/data/simplex_noise.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/inserter/src/algorithm/data/simplex_noise.rs b/inserter/src/algorithm/data/simplex_noise.rs index 23fc007..54eb775 100644 --- a/inserter/src/algorithm/data/simplex_noise.rs +++ b/inserter/src/algorithm/data/simplex_noise.rs @@ -25,8 +25,8 @@ const G3: f64 = 1.0 / 6.0; /// SimplexNoise implementation with DSP-specific permutation. pub struct SimplexNoise { - perm: [i16; 512], - perm_mod12: [i16; 512], + perm: [u8; 512], + perm_mod12: [u8; 512], } impl SimplexNoise { @@ -42,7 +42,7 @@ impl SimplexNoise { /// Initialize with seed-shuffled permutation (using DspRandom). fn init_member_seed(&mut self, seed: i32) { - let mut p: [i16; 256] = std::array::from_fn(|i| i as i16); + let mut p: [u8; 256] = std::array::from_fn(|i| i as u8); // Only shuffle when seed != 0 (matching C# behavior where seed 0 might be special) // Actually the C# always shuffles. Let's shuffle based on seed. @@ -54,10 +54,12 @@ impl SimplexNoise { p[i2] = num; } - for i in 0..512 { - self.perm[i] = p[i & 255]; + for (i, value) in p.into_iter().enumerate() { + self.perm[i] = value; self.perm_mod12[i] = self.perm[i] % 12; } + self.perm.copy_within(..256, 256); + self.perm_mod12.copy_within(..256, 256); } /// 3D simplex noise. From 9cff6d36263112d83c818bb8ff7779bda7319194 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 17:31:02 +0000 Subject: [PATCH 05/18] Reuse duplicate algorithm 9 noise Reuse the first clamped three-octave noise result in the second identical algorithm 9 layer. Simplex noise is immutable after construction, and both calls used the same object and arguments. This removes three noise evaluations per sampled algorithm 9 height and simplifies the code. The full-pipeline result is below the host wall-time noise floor. For seeds 0 through 99, the paired median throughput change was +0.75%, with a bootstrap 95% interval from -5.15% to +5.00%. Median user CPU time improved by 0.98%. The change is retained because it removes guaranteed duplicate work with less code. The canonical 0 through 99 output remained byte-for-byte equal at SHA-256 3f26eb1b456d5a7b71f40a98ab03e0472b05e41cc31e65740dca002da6afa7d6. --- .../algorithm/data/planet_algorithms/algo9.rs | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/inserter/src/algorithm/data/planet_algorithms/algo9.rs b/inserter/src/algorithm/data/planet_algorithms/algo9.rs index 262758c..49dbad4 100644 --- a/inserter/src/algorithm/data/planet_algorithms/algo9.rs +++ b/inserter/src/algorithm/data/planet_algorithms/algo9.rs @@ -99,21 +99,7 @@ impl PlanetAlgorithm for PlanetAlgorithm9 { 2.0, ) * noise_amplitude + noise_offset; - let layer4_noise = self.noise2.noise_3d_fbm( - world_x * (1.0 / 400.0), - world_y * (1.0 / 400.0), - world_z * (1.0 / 400.0), - 3, - 0.5, - 2.0, - ) * noise_amplitude - * noise2_amplitude - + noise2_offset; - let clamped_layer4 = if layer4_noise > 0.0 { - layer4_noise * 0.5 - } else { - layer4_noise - }; + let clamped_layer4 = clamped_layer2; let alt_height = ((layer3_noise + clamped_layer4 + 5.0) * 0.13).powf(6.0) * 24.0 - 24.0; From 7bed4278a95089a9305e60828ccaba49f98600d2 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 17:49:53 +0000 Subject: [PATCH 06/18] Skip unused detail terrain noise Calculate two-octave detail noise in planet algorithms 5 and 6 only when the nonnegative terrain branch consumes it. Noise evaluation is pure. Calls that remain use the same order and arithmetic. For seeds 0 through 99, median user CPU time improved by 1.19%. The paired median wall-throughput change was +0.38%, with a bootstrap 95% interval from -1.69% to +7.97%, so the wall-time effect is inconclusive on this host. The change is retained because it removes work with no additional branch and no added code. The canonical 0 through 99 output remained byte-for-byte equal at SHA-256 3f26eb1b456d5a7b71f40a98ab03e0472b05e41cc31e65740dca002da6afa7d6. --- inserter/src/algorithm/data/planet_algorithms/algo5.rs | 9 ++++----- inserter/src/algorithm/data/planet_algorithms/algo6.rs | 9 ++++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/inserter/src/algorithm/data/planet_algorithms/algo5.rs b/inserter/src/algorithm/data/planet_algorithms/algo5.rs index 524a9d4..595e2b6 100644 --- a/inserter/src/algorithm/data/planet_algorithms/algo5.rs +++ b/inserter/src/algorithm/data/planet_algorithms/algo5.rs @@ -87,13 +87,12 @@ impl PlanetAlgorithm for PlanetAlgorithm5 { 0.0 }; - let detail_noise = self - .noise2 - .noise_3d_fbm(xin * 1.5, yin * 1.5, zin * 1.5, 2, 0.5, 2.0) - .abs(); - let mut terrain_height = height_base - crack_intensity * 1.2 * fluid_clamped; if terrain_height >= 0.0 { + let detail_noise = self + .noise2 + .noise_3d_fbm(xin * 1.5, yin * 1.5, zin * 1.5, 2, 0.5, 2.0) + .abs(); terrain_height += cell_noise * 0.25 + detail_noise * 0.6; } diff --git a/inserter/src/algorithm/data/planet_algorithms/algo6.rs b/inserter/src/algorithm/data/planet_algorithms/algo6.rs index 7aaf9eb..eacd2f3 100644 --- a/inserter/src/algorithm/data/planet_algorithms/algo6.rs +++ b/inserter/src/algorithm/data/planet_algorithms/algo6.rs @@ -87,13 +87,12 @@ impl PlanetAlgorithm for PlanetAlgorithm6 { 0.0 }; - let detail_noise = self - .noise2 - .noise_3d_fbm(xin * 1.5, yin * 1.5, zin * 1.5, 2, 0.5, 2.0) - .abs(); - let mut terrain_height = height_base - crack_intensity * 1.2 * fluid_clamped; if terrain_height >= 0.0 { + let detail_noise = self + .noise2 + .noise_3d_fbm(xin * 1.5, yin * 1.5, zin * 1.5, 2, 0.5, 2.0) + .abs(); terrain_height += cell_noise * 0.25 + detail_noise * 0.6; } From 69365480e6f79716b4940cf80c89d20ade5293a9 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 18:04:25 +0000 Subject: [PATCH 07/18] Write valid PostgreSQL COPY streams Encode the PostgreSQL binary COPY field-length words for all non-null values. Make workers return row fragments, and make each writer add one protocol signature, header field block, and trailer per COPY operation. Increase star-buffer capacity for the valid row size. Also serialize each planet distance from its star, and use that distance for the Dyson-sphere test. The old code used a moon orbital offset for both fields. These two moon values now match their parent planet. This intentionally changes the old malformed wire bytes. The galaxy and vein generation algorithms do not change. PostgreSQL 16 accepted batches of three seeds and a partial final batch for seeds 0 through 9. The result contained 640 stars and 2,446 planets. Unit tests verify protocol field and framing lengths. --- inserter/src/generate_csv.rs | 70 ++++++++++++++++++++++++++---------- inserter/src/threads.rs | 16 ++++++--- 2 files changed, 63 insertions(+), 23 deletions(-) diff --git a/inserter/src/generate_csv.rs b/inserter/src/generate_csv.rs index e4f587c..2cdfd4e 100644 --- a/inserter/src/generate_csv.rs +++ b/inserter/src/generate_csv.rs @@ -3,23 +3,33 @@ use crate::algorithm::data::enums::{PlanetType, ORES}; use crate::algorithm::data::game_desc::GameDesc; use crate::algorithm::generate_stars; -const COPY_HEADER: &[u8] = b"PGCOPY\n\xFF\r\n\0\0"; -const COPY_FOOTER: &[u8] = b"\xFF\xFF"; +pub const COPY_SIGNATURE: &[u8] = b"PGCOPY\n\xFF\r\n\0"; +pub const COPY_HEADER_FIELDS: &[u8] = &[0; 8]; +pub const COPY_FOOTER: &[u8] = b"\xFF\xFF"; + +#[inline] +fn write_row_header(buf: &mut Vec, columns: i16) { + buf.extend_from_slice(&columns.to_be_bytes()); +} #[inline] fn write_i32(buf: &mut Vec, v: i32) { + buf.extend_from_slice(&4i32.to_be_bytes()); buf.extend_from_slice(&v.to_be_bytes()); } #[inline] fn write_i16(buf: &mut Vec, v: i16) { + buf.extend_from_slice(&2i32.to_be_bytes()); buf.extend_from_slice(&v.to_be_bytes()); } #[inline] fn write_f32(buf: &mut Vec, v: f32) { + buf.extend_from_slice(&4i32.to_be_bytes()); buf.extend_from_slice(&v.to_be_bytes()); } #[inline] fn write_bool(buf: &mut Vec, v: bool) { + buf.extend_from_slice(&1i32.to_be_bytes()); buf.push(v as u8); } #[inline] @@ -35,25 +45,16 @@ pub fn gen_formatted(seed: i32) -> anyhow::Result<(Vec, Vec)> { let hab_count = std::cell::Cell::new(0i32); let galaxy = generate_stars(seed, &game_desc, &hab_count); - let mut stars: Vec = Vec::with_capacity(crate::STAR_COUNT * 96); + let mut stars: Vec = Vec::with_capacity(crate::STAR_COUNT * 176); let mut planets: Vec = Vec::with_capacity(crate::STAR_COUNT * 256 * 5); - stars.extend_from_slice(COPY_HEADER); - // flags (4 bytes) + header extension length (4 bytes) - stars.extend_from_slice(&0i32.to_be_bytes()); - stars.extend_from_slice(&0i32.to_be_bytes()); - - planets.extend_from_slice(COPY_HEADER); - planets.extend_from_slice(&0i32.to_be_bytes()); - planets.extend_from_slice(&0i32.to_be_bytes()); - for solar_system in galaxy { let star = solar_system.star.clone(); let star_id = star.index as i32 + seed * 100; // Stars: 22 columns // Columns: id, seed, start_dist, star_index, luminosity, dyson_radius, type, spectr, ore_iron..ore_mag - write_i16(&mut stars, 22); + write_row_header(&mut stars, 22); write_i32(&mut stars, star_id); write_i32(&mut stars, seed); write_f32(&mut stars, star.position.magnitude() as f32); @@ -105,13 +106,17 @@ pub fn gen_formatted(seed: i32) -> anyhow::Result<(Vec, Vec)> { // Columns: star_id, index, orbiting, gas_giant, sun_distance, inside_ds, // satellites, theme_id, gas_h, gas_d, gas_i, tidal_lock, // ore_iron..ore_mag - write_i16(&mut planets, 26); + let sun_distance = planet.get_sun_distance(); + write_row_header(&mut planets, 26); write_i32(&mut planets, star_id); write_i16(&mut planets, planet.index as i16); write_i16(&mut planets, orbiting); write_bool(&mut planets, planet.get_type() == &PlanetType::Gas); - write_f32(&mut planets, planet.get_orbital_radius()); - write_bool(&mut planets, planet.get_orbital_radius() * 40000.0 < star.get_dyson_radius() as f32); + write_f32(&mut planets, sun_distance); + write_bool( + &mut planets, + sun_distance * 40000.0 < star.get_dyson_radius() as f32, + ); write_i16(&mut planets, satellite_count); write_i16(&mut planets, planet.get_theme().id as i16); write_f32(&mut planets, gas_h); @@ -141,8 +146,35 @@ pub fn gen_formatted(seed: i32) -> anyhow::Result<(Vec, Vec)> { } } - stars.extend_from_slice(COPY_FOOTER); - planets.extend_from_slice(COPY_FOOTER); - Ok((stars, planets)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn binary_fields_include_lengths() { + let mut output = Vec::new(); + write_i16(&mut output, 7); + write_i32(&mut output, 11); + write_f32(&mut output, 1.5); + write_bool(&mut output, true); + write_null_col(&mut output); + + assert_eq!( + output, + [ + 0, 0, 0, 2, 0, 7, 0, 0, 0, 4, 0, 0, 0, 11, 0, 0, 0, 4, 63, 192, 0, 0, 0, + 0, 0, 1, 1, 255, 255, 255, 255, + ] + ); + } + + #[test] + fn binary_copy_framing_has_protocol_lengths() { + assert_eq!(COPY_SIGNATURE.len(), 11); + assert_eq!(COPY_HEADER_FIELDS.len(), 8); + assert_eq!(COPY_FOOTER, [255, 255]); + } +} diff --git a/inserter/src/threads.rs b/inserter/src/threads.rs index 65d0530..1908f30 100644 --- a/inserter/src/threads.rs +++ b/inserter/src/threads.rs @@ -5,7 +5,9 @@ use std::io::Write; use crossbeam_channel::{Receiver, RecvTimeoutError, Sender}; use postgres::{Client, NoTls}; use anyhow::Result; -use crate::generate_csv::gen_formatted; +use crate::generate_csv::{ + gen_formatted, COPY_FOOTER, COPY_HEADER_FIELDS, COPY_SIGNATURE, +}; use crate::{log_info, COMMITTED_SEEDS, COMMIT_COUNT, DB_STR, PROGRESS_WORKERS}; use crate::misc::{COPY_PLANET, COPY_STAR}; @@ -36,17 +38,23 @@ pub fn commit_thread(rec: Receiver<(Vec, Vec)>) -> Result<()> { { let mut scpy = txn.copy_in(COPY_STAR)?; + scpy.write_all(COPY_SIGNATURE)?; + scpy.write_all(COPY_HEADER_FIELDS)?; for (star, _) in &batch { - scpy.write_all(star).expect("writing to scpy failed"); + scpy.write_all(star)?; } + scpy.write_all(COPY_FOOTER)?; scpy.finish()?; } { let mut pcpy = txn.copy_in(COPY_PLANET)?; + pcpy.write_all(COPY_SIGNATURE)?; + pcpy.write_all(COPY_HEADER_FIELDS)?; for (_, planet) in &batch { - pcpy.write_all(planet).expect("writing to pcpy failed"); + pcpy.write_all(planet)?; } + pcpy.write_all(COPY_FOOTER)?; pcpy.finish()?; } @@ -65,4 +73,4 @@ pub fn writer_sink(rec: Receiver<(Vec, Vec)>) -> Result<()> { } } Ok(()) -} \ No newline at end of file +} From 9966dcf8db57b82d6cc6e6539315fb7ca07fae7f Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 18:13:10 +0000 Subject: [PATCH 08/18] Handle writer input lulls Block until the first batch item is available, flush a partial database batch after a one-second lull, and exit normally when the channel closes. Reuse the batch allocation and use relaxed ordering for the display-only committed counter. Benchmark sinks now use blocking receives. Generation-only performance did not improve. For seeds 0 through 99, the paired median throughput change was -1.49%, with a bootstrap 95% interval from -6.72% to +0.87%. Median user CPU time changed by -0.28%. The change is retained to prevent timeout panics and preserve partial work. A PostgreSQL integration run with five seeds, a three-seed batch target, and a partial final batch completed successfully. --- inserter/src/threads.rs | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/inserter/src/threads.rs b/inserter/src/threads.rs index 1908f30..202fe83 100644 --- a/inserter/src/threads.rs +++ b/inserter/src/threads.rs @@ -23,14 +23,24 @@ pub fn worker_thread(seeds: Range, send: Sender<(Vec, Vec)>, id: us } pub fn commit_thread(rec: Receiver<(Vec, Vec)>) -> Result<()> { let mut client = Client::connect(&*DB_STR.as_str(), NoTls)?; + let mut batch: Vec<(Vec, Vec)> = Vec::with_capacity(*COMMIT_COUNT); - 'outer: loop { - let mut batch: Vec<(Vec, Vec)> = Vec::with_capacity(*COMMIT_COUNT); - 'inner: for i in 0..*COMMIT_COUNT { + loop { + batch.clear(); + match rec.recv() { + Ok(msg) => batch.push(msg), + Err(_) => break, + } + + let mut disconnected = false; + for _ in 1..*COMMIT_COUNT { match rec.recv_timeout(Duration::new(1, 0)) { Ok(msg) => batch.push(msg), - Err(RecvTimeoutError::Timeout) => panic!("commit_thread: recv_timeout reached - channel stall detected (>1s lull)"), - Err(RecvTimeoutError::Disconnected) => if i == 0 {break 'outer} else {break 'inner}, + Err(RecvTimeoutError::Timeout) => break, + Err(RecvTimeoutError::Disconnected) => { + disconnected = true; + break; + } } } @@ -59,18 +69,15 @@ pub fn commit_thread(rec: Receiver<(Vec, Vec)>) -> Result<()> { } txn.commit()?; - COMMITTED_SEEDS.fetch_add(batch.len() as i32, std::sync::atomic::Ordering::SeqCst); + COMMITTED_SEEDS.fetch_add(batch.len() as i32, Relaxed); + if disconnected { + break; + } } log_info!("Writer thread terminated"); Ok(()) } pub fn writer_sink(rec: Receiver<(Vec, Vec)>) -> Result<()> { - loop { - match rec.recv_timeout(Duration::new(1, 0)) { - Ok(_) => {}, - Err(RecvTimeoutError::Timeout) => panic!("writer_sink: recv_timeout reached - channel stall detected (>1s lull)"), - Err(RecvTimeoutError::Disconnected) => break, - } - } + while rec.recv().is_ok() {} Ok(()) } From 795b543763e27cd7fceb74c89e245e9ffb363d14 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 18:27:57 +0000 Subject: [PATCH 09/18] Reduce pipeline buffer defaults Use one writer, 64 seeds per transaction, and a 64-entry channel by default. Remove an unused and incorrect buffer estimate. Environment variables can still override all values. At 32 workers on the 500-seed PostgreSQL workload, the prior defaults produced 6.568 seeds per second and the new settings produced 6.595 seeds per second. This 0.4% difference is not material. The configured completed-buffer bound falls from about 5,000 seeds to 128 seeds, which reduces worst-case retained payload memory and database connections. --- inserter/src/main.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/inserter/src/main.rs b/inserter/src/main.rs index ed771aa..1fe92d5 100644 --- a/inserter/src/main.rs +++ b/inserter/src/main.rs @@ -43,9 +43,9 @@ lazy_static! { static ref START_SEED: i32 = env_int!("START_SEED", 0); static ref END_SEED: i32 = env_int!("END_SEED", 10_000); static ref WORKER_THREADS: i32 = env_int!("WORKER_THREADS", 8); - static ref WRITER_THREADS: i32 = env_int!("WRITER_THREADS", 4); - static ref COMMIT_COUNT: usize = env_int!("COMMIT_COUNT", 1000) as usize; - static ref CHANNEL_SIZE: usize = env_int!("CHANNEL_SIZE", 1000) as usize; + static ref WRITER_THREADS: i32 = env_int!("WRITER_THREADS", 1); + static ref COMMIT_COUNT: usize = env_int!("COMMIT_COUNT", 64) as usize; + static ref CHANNEL_SIZE: usize = env_int!("CHANNEL_SIZE", 64) as usize; static ref CHECKPOINT_FILE: String = env_str!("CHECKPOINT_FILE", "checkpoints.txt"); static ref OVERRIDE_CHECKPOINTS: bool = env_bool!("OVERRIDE_CHECKPOINTS"); @@ -74,7 +74,6 @@ lazy_static! { PG_DBNAME.as_str() ) }; - static ref MAX_BUFFER: usize = *CHANNEL_SIZE + *COMMIT_COUNT * *WORKER_THREADS as usize - 2; static ref LOG_NAME: String = { let num = std::fs::read_dir(&*LOG_DIR).map(|x| { x.map(|x| x.unwrap().file_name().into_string().unwrap()) From 38f40a80225d46ce5a2400ec67ff4642ee40001c Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 18:28:49 +0000 Subject: [PATCH 10/18] Adapt the default worker count Use up to 32 workers by default and cap the value at the number of requested seeds. Explicit WORKER_THREADS values still take precedence. More ranges reduce the long tail from unequal seed-generation costs without changing any per-seed output. On the two-logical-CPU test host with PostgreSQL enabled, the 500-seed workload increased from 5.962 seeds per second at the prior default of 8 workers to 6.595 seeds per second at 32 workers. This is a 10.6% throughput gain. Peak generator RSS increased from 18 MiB to 33 MiB. --- inserter/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/inserter/src/main.rs b/inserter/src/main.rs index 1fe92d5..ede1dcd 100644 --- a/inserter/src/main.rs +++ b/inserter/src/main.rs @@ -42,7 +42,10 @@ static PROGRESS_WORKERS: [AtomicI32; MAX_WORKERS] = [const { AtomicI32::new(0) } lazy_static! { static ref START_SEED: i32 = env_int!("START_SEED", 0); static ref END_SEED: i32 = env_int!("END_SEED", 10_000); - static ref WORKER_THREADS: i32 = env_int!("WORKER_THREADS", 8); + static ref WORKER_THREADS: i32 = std::env::var("WORKER_THREADS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or_else(|| (*END_SEED - *START_SEED).clamp(1, MAX_WORKERS as i32)); static ref WRITER_THREADS: i32 = env_int!("WRITER_THREADS", 1); static ref COMMIT_COUNT: usize = env_int!("COMMIT_COUNT", 64) as usize; static ref CHANNEL_SIZE: usize = env_int!("CHANNEL_SIZE", 64) as usize; From 1b72146fb52ecf0ab01e4f1b1ff0a6fb05637983 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 18:29:13 +0000 Subject: [PATCH 11/18] Redact database connection logs Log the PostgreSQL host, port, database, and user instead of the complete connection URI. This prevents PG_PASS from appearing in console logs. The database connection and generated output do not change. --- inserter/src/misc.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/inserter/src/misc.rs b/inserter/src/misc.rs index a741d5d..341b498 100644 --- a/inserter/src/misc.rs +++ b/inserter/src/misc.rs @@ -95,8 +95,11 @@ pub fn create_db_schema() -> bool { pub fn check_db_connection() -> bool { crate::log_info!( - "Checking database connection to {}", - *crate::DB_STR + "Checking PostgreSQL connection to {}:{}/{} as {}", + *crate::PG_NETLOC, + *crate::PG_PORT, + *crate::PG_DBNAME, + *crate::PG_USER, ); match postgres::Client::connect((*crate::DB_STR).as_str(), postgres::NoTls) { @@ -213,4 +216,4 @@ macro_rules! process_option { ($user_option:expr) => { $user_option.to_ascii_lowercase().replace(" ", "").replace("_", "").replace("-", "") }; -} \ No newline at end of file +} From ae0663b72ae3799ec33eeaf81b47109f105d0fe6 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 19:13:20 +0000 Subject: [PATCH 12/18] Prepare the 0.7.0 image Set the Rust package version to 0.7.0. Pin the builder to Rust 1.97.0, require the lock file, add Open Container Initiative image labels, and restrict the Docker build context to required files. Ignore local PostgreSQL data and remove one unused import. A clean Linux AMD64 image build completed successfully. The scratch image is 923,982 bytes and contains the expected 0.7.0 version and source-revision labels. --- .dockerignore | 4 ++++ .gitignore | 3 ++- Dockerfile | 13 +++++++++++-- inserter/Cargo.lock | 2 +- inserter/Cargo.toml | 3 +-- inserter/src/algorithm/data/planet.rs | 2 -- 6 files changed, 19 insertions(+), 8 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d51eaf5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +** +!Dockerfile +!inserter/ +!inserter/** diff --git a/.gitignore b/.gitignore index 9797cdc..5c00bd7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /target /pkg +/data/ # Editor directories and files .idea @@ -37,4 +38,4 @@ env.bak/ venv.bak/ # AI agent personal preferences (per-developer, not shared) -.ai/local.md \ No newline at end of file +.ai/local.md diff --git a/Dockerfile b/Dockerfile index 2b9f661..2d921b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,18 @@ -FROM rust:latest AS builder +ARG IMAGE_VERSION=dev +ARG VCS_REF=unknown + +FROM rust:1.97.0@sha256:b92b8c8574f8f3b207fcb0912fb3e2de4041580b5934d90312d53938c9a038a9 AS builder WORKDIR /app COPY ./inserter/ . RUN rustup target add x86_64-unknown-linux-musl -RUN cargo build --release --target x86_64-unknown-linux-musl --bin dsp_seed_finder +RUN cargo build --release --locked --target x86_64-unknown-linux-musl --bin dsp_seed_finder FROM scratch +ARG IMAGE_VERSION +ARG VCS_REF +LABEL org.opencontainers.image.title="DSP Seed Finder" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.revision="${VCS_REF}" \ + org.opencontainers.image.source="https://github.com/SuperB3333/DSPSeedDatabase" COPY --from=builder /app/target/x86_64-unknown-linux-musl/release/dsp_seed_finder /app/app CMD ["/app/app"] diff --git a/inserter/Cargo.lock b/inserter/Cargo.lock index dec3197..c762436 100644 --- a/inserter/Cargo.lock +++ b/inserter/Cargo.lock @@ -210,7 +210,7 @@ dependencies = [ [[package]] name = "dsp_seed_finder" -version = "0.1.0" +version = "0.7.0" dependencies = [ "anyhow", "crossbeam-channel", diff --git a/inserter/Cargo.toml b/inserter/Cargo.toml index 89ab8e0..537b127 100644 --- a/inserter/Cargo.toml +++ b/inserter/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dsp_seed_finder" -version = "0.1.0" +version = "0.7.0" edition = "2021" [profile.release] @@ -24,4 +24,3 @@ crossterm = "0.29.0" anyhow = "1.0.103" lazy_static = "1.5.0" itertools = "0.15.0" - diff --git a/inserter/src/algorithm/data/planet.rs b/inserter/src/algorithm/data/planet.rs index 903e23a..dbebec7 100644 --- a/inserter/src/algorithm/data/planet.rs +++ b/inserter/src/algorithm/data/planet.rs @@ -2,8 +2,6 @@ use crate::algorithm::data::birth_points::BirthPoints; use crate::algorithm::data::game_desc::GameDesc; use crate::algorithm::data::planet_raw_data::PlanetRawData; use crate::algorithm::data::vector_f2::VectorF2; -use std::convert::TryFrom; - use super::enums::{OceanType, PlanetType, SpectrType, StarType, ThemeDistribute, VeinType}; use super::pose::Pose; use super::quaternion::Quaternion; From 9606e0d4bfdc5ea3306fa65a9ab957bc1ce651ab Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 19:14:58 +0000 Subject: [PATCH 13/18] Update the 0.7.0 stack Use the 0.7.0 generator image in all Compose files. Pin PostgreSQL 16 and PostgREST 12 images, require the database password, bind published ports to loopback, and use service health and completion dependencies in the full stack. Replace the unavailable Python API image and broken unused prototype with PostgREST. The generator now runs as a finite job and connects to PostgreSQL through the Compose service network. Docker Compose validated all three configurations successfully with a supplied password. Blank-state service verification follows in the release procedure. --- api/main.py | 30 ----------------- api/query_database.py | 23 ------------- api/requirements.txt | 4 --- compose/compose.postgres.yaml | 8 ++--- compose/compose.yaml | 11 ++++--- compose/compose_full.yaml | 39 ++++++++++++++--------- compose/init/01-create-postgrest-user.sql | 5 --- 7 files changed, 34 insertions(+), 86 deletions(-) delete mode 100644 api/main.py delete mode 100644 api/query_database.py delete mode 100644 api/requirements.txt delete mode 100644 compose/init/01-create-postgrest-user.sql diff --git a/api/main.py b/api/main.py deleted file mode 100644 index ae7a34e..0000000 --- a/api/main.py +++ /dev/null @@ -1,30 +0,0 @@ -from fastapi import FastAPI -from hashlib import sha256 - -from query_database import query_database - -import time - -app = FastAPI() - -glob_return_dict = {} - -@app.post("/start_query") -def start_query(query: str, params: list): - h = sha256(query.encode()) - h.update(bytes(int(time.time()))) - qid = h.hexdigest() - glob_return_dict[qid] = None - - query_database((query, params), qid) - - return qid - -@app.post("/query_ready") -def query_status(qid: str): - return "true" if glob_return_dict[qid] is not None else "false" -@app.post("/get_result") -def query_result(qid: str): - res = list(glob_return_dict[qid]) - glob_return_dict[qid] = None - return res diff --git a/api/query_database.py b/api/query_database.py deleted file mode 100644 index 8ab55b4..0000000 --- a/api/query_database.py +++ /dev/null @@ -1,23 +0,0 @@ -import threading, psycopg2, os - - - -def _new_connection(): - return psycopg2.connect(database=os.getenv("PG_DBNAME"), user=os.getenv("PG_USER"), password=os.getenv("PG_PASS"), host=os.getenv("PG_NETLOC"), port=os.getenv("PG_PORT")) - -_client = None -def _get_client(): - global _client - if _client is not None: return _client - _client = _new_connection() - return _client - -def query_database(query: tuple[str, list], qid: str) -> None: - threading.Thread(target=_query_database, args=(query + "\nLIMIT 100", qid), name=f"Database query {qid}", daemon=True) -def _query_database(query: tuple[str, list], qid: str) -> None: - global glob_return_dict - cur = _get_client().cursor() - cur.execute(*query) - res = cur.fetchall() - - glob_return_dict[qid] = map(lambda x: x[0], res) \ No newline at end of file diff --git a/api/requirements.txt b/api/requirements.txt deleted file mode 100644 index 6270e8f..0000000 --- a/api/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -fastapi -uvicorn -python-multipart -psycopg2 \ No newline at end of file diff --git a/compose/compose.postgres.yaml b/compose/compose.postgres.yaml index 96fc88d..a9cc8fb 100644 --- a/compose/compose.postgres.yaml +++ b/compose/compose.postgres.yaml @@ -1,18 +1,18 @@ services: postgres: - image: postgres:16 + image: postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 container_name: postgres environment: POSTGRES_USER: postgres - POSTGRES_PASSWORD: rootpassword + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} POSTGRES_DB: dsp volumes: - "../data:/var/lib/postgresql/data" ports: - - "5432:5432" + - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" restart: unless-stopped healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] interval: 5s timeout: 5s retries: 5 diff --git a/compose/compose.yaml b/compose/compose.yaml index ad62536..7d847f2 100644 --- a/compose/compose.yaml +++ b/compose/compose.yaml @@ -1,17 +1,18 @@ services: seedfinder: - image: toti330/dsp_seed_finder:0.6.0 + image: toti330/dsp_seed_finder:0.7.0 container_name: seedfinder environment: - END_SEED: 1000 # max is 10,000,000 (10m) + START_SEED: ${START_SEED:-0} + END_SEED: ${END_SEED:-1000} NO_TUI: 1 PG_USER: postgres - PG_PASS: rootpassword # Default values + PG_PASS: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} PG_PORT: 5432 PG_DBNAME: dsp PG_NETLOC: host.docker.internal - LOG_LEVEL: debug + LOG_LEVEL: ${LOG_LEVEL:-info} network_mode: bridge - restart: on-failure + restart: "no" extra_hosts: - "host.docker.internal:host-gateway" diff --git a/compose/compose_full.yaml b/compose/compose_full.yaml index a650379..a0bc8f4 100644 --- a/compose/compose_full.yaml +++ b/compose/compose_full.yaml @@ -1,39 +1,48 @@ services: postgres: - image: postgres:16 + image: postgres:16@sha256:33f923b05f64ca54ac4401c01126a6b92afe839a0aa0a52bc5aeb5cc958e5f20 container_name: postgres environment: POSTGRES_USER: postgres - POSTGRES_PASSWORD: rootpassword + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} POSTGRES_DB: dsp volumes: - "../data:/var/lib/postgresql/data" ports: - - "5432:5432" + - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" restart: unless-stopped healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] interval: 5s timeout: 5s retries: 5 seedfinder: - image: toti330/seedfinder:0.6.0 + image: toti330/dsp_seed_finder:0.7.0 container_name: seedfinder environment: - END_SEED: 100000 # max is 10,000,000 (10m) - NO_TUI: 1 # Disable progress tui + START_SEED: ${START_SEED:-0} + END_SEED: ${END_SEED:-100} + NO_TUI: 1 PG_USER: postgres - PG_PASS: rootpassword # Default values + PG_PASS: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} PG_PORT: 5432 PG_DBNAME: dsp - PG_NETLOC: host.docker.internal - restart: on-failure + PG_NETLOC: postgres + LOG_LEVEL: ${LOG_LEVEL:-info} + restart: "no" depends_on: - - postgres + postgres: + condition: service_healthy api: - image: superb3333/postgres_api + image: postgrest/postgrest:v12.2.12@sha256:5f4ce744539bbba786b4e24dbbd95bdb2a956dcf568c5374995a0ff4a68f5bd2 container_name: api environment: - PG_DBNAME: dsp - PG_USER: api_user - PG_PASS: api_password + PGRST_DB_URI: postgres://api_user:api_password@postgres:5432/dsp + PGRST_DB_SCHEMA: public + PGRST_DB_ANON_ROLE: api_user + ports: + - "127.0.0.1:${API_PORT:-3000}:3000" + restart: unless-stopped + depends_on: + seedfinder: + condition: service_completed_successfully diff --git a/compose/init/01-create-postgrest-user.sql b/compose/init/01-create-postgrest-user.sql deleted file mode 100644 index ae99b50..0000000 --- a/compose/init/01-create-postgrest-user.sql +++ /dev/null @@ -1,5 +0,0 @@ -CREATE USER postgrest_api WITH PASSWORD 'postgrest_api_password'; - -GRANT USAGE ON SCHEMA public TO postgrest_api; -GRANT SELECT ON stars TO postgrest_api; -GRANT SELECT ON planets TO postgrest_api; From 35f36bb51aafecc76a65817e18b80dfd09b9b174 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 19:58:37 +0000 Subject: [PATCH 14/18] Document the performance release Rewrite the README and Compose guide in ASD-STE100 Simplified Technical English. Add reproducible build, operation, reset, configuration, image, and API procedures. Add the controlled benchmark method, profile results, all retained and reverted experiments, PostgreSQL scaling data, deterministic hashes, target limits, and the stop decision. Add raw final observations and a machine-readable experiment record in JSON. The final one-worker comparison measured a 38.0% paired median throughput gain with a bootstrap 95% interval from 32.0% to 39.5%. Two 1,000-seed outputs were byte equal. --- README.md | 214 +++++++++++++---- .../results/2026-07-29-experiments.json | 171 ++++++++++++++ benchmarks/results/2026-07-29-final.json | 223 ++++++++++++++++++ compose/compose.md | 105 ++++++--- docs/performance.md | 206 ++++++++++++++++ 5 files changed, 842 insertions(+), 77 deletions(-) create mode 100644 benchmarks/results/2026-07-29-experiments.json create mode 100644 benchmarks/results/2026-07-29-final.json create mode 100644 docs/performance.md diff --git a/README.md b/README.md index 6a534ab..0b7c39c 100644 --- a/README.md +++ b/README.md @@ -1,74 +1,194 @@ # DSP Seed Finder -A high-performance pipeline for generating, indexing, and searching Dyson Sphere Program (DSP) galaxy seeds. It replaces the trial-and-error of seed hunting with a multi-threaded Rust generator and a flexible Python-based SQL search engine. +DSP Seed Finder generates galaxy data for Dyson Sphere Program (DSP). A Rust +program writes the data to PostgreSQL. PostgREST supplies a read-only +application programming interface (API) that uses Hypertext Transfer Protocol +(HTTP). -## Core Capabilities +## Services -- **High-Speed Generation:** Multi-threaded Rust implementation of the DSP galaxy generation algorithm, capable of processing thousands of seeds per second. -- **SQL-Powered Search:** Exports seed data to PostgreSQL for complex queries that the game's UI cannot perform (e.g., "Find a seed with 6 O-stars and a tidally locked planet inside the Dyson radius"). -- **Resume-Safe:** Atomic checkpointing ensures that long-running generation jobs (e.g., processing 10 million seeds) can be paused and resumed without data corruption or duplication. -- **Flexible Scoring:** A Python scoring engine ranks seeds based on weighted metrics like total luminosity, rare ore availability, or "inside-sphere" planet counts. +- `seedfinder` generates stars, planets, gases, and vein amounts. +- `postgres` stores the generated data. +- `api` gives read-only access to the `stars` and `planets` tables. -## Quick Start +The generator processes the configured seed range. The generator then stops. + +## Requirements + +Install these tools: + +- Docker Engine +- Docker Compose +- `curl` for the API check + +The release platform is Linux on a 64-bit x86 processor. + +## Start The Full Stack + +1. Set a PostgreSQL password. + + ```bash + export POSTGRES_PASSWORD='replace-this-password' + ``` + +2. Start the full stack with the default 100-seed range. + + ```bash + docker compose -f compose/compose_full.yaml up -d --wait + ``` + +3. Make sure that the generator exit code is `0`. + + ```bash + docker inspect seedfinder --format '{{.State.ExitCode}}' + ``` + +4. Read one star from the API. + + ```bash + curl --fail 'http://127.0.0.1:3000/stars?select=seed,star_index&limit=1' + ``` + +Set `END_SEED` to the first seed that the generator does not process. -### 1. Infrastructure -The fastest way to get the database ready is via Docker Compose: ```bash -docker compose -f compose/compose.postgres.yaml up -d +END_SEED=1000 docker compose -f compose/compose_full.yaml up -d --wait ``` -### 2. Schema Setup -Initialize the database schema (requires `psycopg2`): +## Start Services Independently + +Set the PostgreSQL password: + ```bash -python3 create_database.py +export POSTGRES_PASSWORD='replace-this-password' ``` -### 3. Generate Seeds -Build and run the generator. This will populate the database with seeds in the specified range. +Start PostgreSQL: + ```bash -# Configuration via environment variables -START_SEED=0 END_SEED=100000 WORKER_THREADS=16 cargo run --release +docker compose -f compose/compose.postgres.yaml up -d --wait ``` -### 4. Search & Score -Find the best seeds based on your own criteria: +Start the generator after the PostgreSQL health check passes: + ```bash -# Example: Rank seeds by O-star luminosity and Silicon availability -python3 score_seeds.py --weight luminosity=1.5 --weight ore_silicium=1.0 --top 10 +POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ +docker compose -f compose/compose.yaml up ``` -## Architecture +The standalone generator connects to PostgreSQL on the Docker host through +`host.docker.internal`. + +## Configuration -### The Pipeline -1. **Generation (`src/`)**: A Rust crate that replicates the game's internal generation logic. It bypasses all rendering/UI and uses `crossbeam-channel` to pipe generated CSV data from worker threads to database writer threads. -2. **Ingestion**: Uses PostgreSQL `COPY FROM STDIN` for maximum throughput. Tables are created as `UNLOGGED` by default to minimize Write-Ahead Log (WAL) overhead. -3. **Search (`rules.py`, `score_seeds.py`)**: A Python layer that compiles high-level search criteria into optimized SQL. +The generator reads these environment variables: -### Database Schema -Data is split into two primary tables: -- **`stars`**: High-level system data (luminosity, type, star-level aggregated ores). -- **`planets`**: Detailed planet-level data (vein counts, tidal locking, orbital radius, gas rates). +| Variable | Executable default | Function | +| --- | ---: | --- | +| `START_SEED` | `0` | Set the first seed. | +| `END_SEED` | `10000` | Set the first seed that the generator does not process. | +| `WORKER_THREADS` | Adaptive | Override the generator worker count. | +| `WRITER_THREADS` | `1` | Set the PostgreSQL writer count. | +| `COMMIT_COUNT` | `64` | Set the maximum seeds in one transaction. | +| `CHANNEL_SIZE` | `64` | Set the completed-seed queue capacity. | +| `BENCHMARK` | Off | Discard generated output. | +| `NO_TUI` | Off | Disable the terminal user interface (TUI). | +| `PG_NETLOC` | `localhost` | Set the PostgreSQL host. | +| `PG_PORT` | `5432` | Set the PostgreSQL port. | +| `PG_DBNAME` | `dsp` | Set the PostgreSQL database. | +| `PG_USER` | `postgres` | Set the PostgreSQL user. | +| `PG_PASS` | `rootpassword` | Set the PostgreSQL password. | -**Note on Sentinels:** To handle the variance in planet types, the database uses `-1` as a sentinel value for ores that do not exist on a given planet (e.g., iron on a gas giant). The scoring engine uses `GREATEST(estimate, 0)` to ensure these sentinels never subtract from a seed's score. +The adaptive worker count is the smaller value of the seed count and 32. Set +`WORKER_THREADS` to select a different worker count. -## Configuration +The full-stack Compose file sets `END_SEED` to `100`. The standalone generator +Compose file sets `END_SEED` to `1000`. Each generator Compose file enables +`NO_TUI`. + +## Build And Test + +Build the release image: + +```bash +docker build \ + --build-arg IMAGE_VERSION=0.7.0 \ + --build-arg VCS_REF="$(git rev-parse HEAD)" \ + --tag dsp-seed-finder:0.7.0 . +``` + +Run the Rust tests in a container: + +```bash +docker run --rm \ + --mount type=bind,source="$PWD/inserter",target=/src,readonly \ + --workdir /src \ + rust:1.97.0 \ + cargo test --release --locked +``` + +## Performance + +The controlled test used one worker. The test measured the paired change in +throughput. The median gain was 38.0% for version 0.7.0. Median throughput +increased from 2.649 seeds/s to 3.668 seeds/s. The bootstrap interval was 32.0% +to 39.5% at 95% confidence. + +For the method, raw data, discarded experiments, and scaling results, refer to +[`docs/performance.md`](docs/performance.md). + +## Determinism + +A determinism test ran the same version 0.7.0 executable two times. The test +processed seeds from 0 to 999. The `cmp` tool found no different bytes. Each +file contained 50,069,759 bytes. + +```text +d188c78a555c95f2188db893a8c9890ec2a50848c9b8826f6e72c7104b3b7acd +``` + +The value is a 256-bit Secure Hash Algorithm (SHA-256) digest. The result is +valid for the tested executable. GNU and musl math libraries can give different +planet values. -The Rust generator is configured via environment variables: -- `START_SEED` / `END_SEED`: The range of seeds to process. -- `WORKER_THREADS`: Number of threads generating galaxy data. -- `WRITER_THREADS`: Number of threads pushing data to Postgres. -- `CHECKPOINT_FILE`: Path to the progress tracking file (default: `checkpoints.txt`). -- `BENCHMARK=1`: Disables DB writes to test pure generation throughput. +## Data Storage -## Sharp Edges +The database uses PostgreSQL `UNLOGGED` tables. PostgreSQL does not restore data +from these tables after a crash. Use logged tables if crash recovery is +necessary. -- **Checkpoint Invariants:** The generator bails out if it detects a mismatch between the current config and an existing `checkpoints.txt`. If you change your seed range or worker count, you must delete the old checkpoint file. -- **UNLOGGED Tables:** For speed, tables are `UNLOGGED`. If the database crashes, data may be lost. Run `ALTER TABLE ... SET LOGGED` if you require durability. -- **Database Load:** The generator can easily saturate a standard Postgres installation. If you see "channel stall" warnings, try increasing `COMMIT_COUNT` or decreasing `WORKER_THREADS`. +The database files are in `data/`. -## Making Your First Change +1. Make sure that the current directory is the repository root. -The shortest path to a meaningful change is adding a new metric to the scoring engine: -1. Open `score_seeds.py`. -2. Add a new entry to the `_build_metrics` dictionary (e.g., a filter for specific planet themes). -3. Run the script with your new metric: `--weight your_metric=1.0`. + ```bash + test -f Dockerfile + ``` + +2. Stop all services. + + ```bash + docker compose -f compose/compose_full.yaml down --remove-orphans + ``` + +3. Remove the database directory. + + ```bash + sudo rm -rf -- data + ``` + +4. Create an empty database directory. + + ```bash + mkdir data + ``` + +## Release Image + +The release image is `toti330/dsp_seed_finder:0.7.0`. + +Examine the remote image: + +```bash +docker buildx imagetools inspect toti330/dsp_seed_finder:0.7.0 +``` diff --git a/benchmarks/results/2026-07-29-experiments.json b/benchmarks/results/2026-07-29-experiments.json new file mode 100644 index 0000000..2a4dd3a --- /dev/null +++ b/benchmarks/results/2026-07-29-experiments.json @@ -0,0 +1,171 @@ +{ + "schema_version": 1, + "date": "2026-07-29", + "baseline_commit": "c0ed162ddf820b8d4ada0127b597cd64a8e35a88", + "platform": "linux/amd64", + "rust_version": "1.97.0", + "system": { + "cpu": "Intel Xeon Platinum 8370C at 2.80 GHz", + "logical_processors": 2, + "physical_cores": 1, + "memory_gib": 7.7, + "kernel": "Linux 6.17.0-1021-azure", + "docker_version": "29.6.1" + }, + "profile": { + "baseline_sample_percent": { + "simplex_noise_3d": 45.39, + "page_fault_handling": 15.55, + "terrain_height_interpolation": 10.56 + }, + "optimized_sample_percent": { + "simplex_noise_3d": 56.2, + "terrain_height_interpolation": 10.98 + }, + "baseline_100_seed_minor_page_faults": 3319980, + "pooled_cache_100_seed_minor_page_faults": 7619, + "hardware_counters_supported": false + }, + "experiments": [ + { + "name": "pooled-terrain-cache", + "decision": "keep", + "median_change_percent": 27.1381, + "bootstrap_95_percent_ci": [16.5794, 31.3509], + "output_equal": true + }, + { + "name": "spacing-before-terrain", + "decision": "keep", + "median_change_percent": 5.1378, + "bootstrap_95_percent_ci": [2.6168, 8.3876], + "output_equal": true + }, + { + "name": "compact-permutations", + "decision": "keep", + "median_change_percent": 2.9471, + "bootstrap_95_percent_ci": [0.9125, 7.5649], + "output_equal": true + }, + { + "name": "fused-algorithm-12-noise", + "decision": "revert", + "median_change_percent": 0.1962, + "bootstrap_95_percent_ci": [-4.7622, 0.7467], + "output_equal": true + }, + { + "name": "reuse-algorithm-9-noise", + "decision": "keep", + "median_change_percent": 0.7536, + "bootstrap_95_percent_ci": [-5.1496, 4.9991], + "median_user_cpu_change_percent": 0.9846, + "output_equal": true + }, + { + "name": "conditional-algorithm-10-noise", + "decision": "revert", + "median_change_percent": -0.9184, + "bootstrap_95_percent_ci": [-2.9559, 4.2627], + "output_equal": true + }, + { + "name": "conditional-algorithm-5-and-6-noise", + "decision": "keep", + "median_change_percent": 0.3772, + "bootstrap_95_percent_ci": [-1.6933, 7.9741], + "median_user_cpu_change_percent": 1.1884, + "output_equal": true + }, + { + "name": "partial-algorithm-4-distance", + "decision": "revert", + "median_change_percent": -1.8394, + "bootstrap_95_percent_ci": [-8.9872, -0.1795], + "output_equal": true + }, + { + "name": "blocking-writer-input", + "decision": "keep-for-reliability", + "median_change_percent": -1.4859, + "bootstrap_95_percent_ci": [-6.7233, 0.8698] + }, + { + "name": "small-pipeline-defaults", + "decision": "keep", + "baseline_seeds_per_second": 6.5683, + "candidate_seeds_per_second": 6.5949, + "seed_count": 500 + }, + { + "name": "adaptive-worker-count", + "decision": "keep", + "baseline_workers": 8, + "candidate_workers": 32, + "baseline_seeds_per_second": 5.9618, + "candidate_seeds_per_second": 6.5949, + "seed_count": 500 + }, + { + "name": "final-one-worker-comparison", + "decision": "keep", + "median_change_percent": 37.9963, + "bootstrap_95_percent_ci": [31.9682, 39.4965], + "seed_count": 100 + } + ], + "database_scaling": { + "seed_count": 500, + "postgres_version": "16", + "writer_count": 1, + "commit_count": 64, + "samples": [ + {"workers": 2, "seeds_per_second": 4.538395, "max_rss_kib": 14432}, + {"workers": 8, "seeds_per_second": 5.961821, "max_rss_kib": 18008}, + {"workers": 16, "seeds_per_second": 6.179457, "max_rss_kib": 23876}, + {"workers": 32, "seeds_per_second": 6.594936, "max_rss_kib": 33808} + ], + "four_writer_seeds_per_second": 6.589842, + "baseline_scale_status": "failed after writer sink timeout" + }, + "correctness": { + "pre_copy_seed_range": [0, 100], + "pre_copy_sha256": "3f26eb1b456d5a7b71f40a98ab03e0472b05e41cc31e65740dca002da6afa7d6", + "copy_acceptance": { + "seed_count": 10, + "batch_size": 3, + "star_rows": 640, + "planet_rows": 2446 + }, + "final_repeat": { + "seed_range": [0, 1000], + "runs": 2, + "bytes": 50069759, + "sha256": "d188c78a555c95f2188db893a8c9890ec2a50848c9b8826f6e72c7104b3b7acd" + }, + "database_worker_comparison": { + "seed_range": [0, 500], + "worker_counts": [8, 32], + "star_rows": 32000, + "planet_rows": 122266, + "star_hash": "51ddf9bbe5e3a6dbf17dfa5e65f89621", + "planet_hash": "32096b2ffd8476930b437c1b9711cb50" + }, + "gnu_and_musl_planet_hashes_equal": false + }, + "not_tested": [ + { + "name": "unsafe-fast-math-or-simd", + "reason": "Can change terrain comparisons and random-number consumption" + }, + { + "name": "profile-guided-optimization", + "reason": "Binds code layout to one training corpus and hardware profile" + }, + { + "name": "dynamic-seed-queue", + "reason": "Requires a new checkpoint protocol for in-progress seeds" + } + ] +} diff --git a/benchmarks/results/2026-07-29-final.json b/benchmarks/results/2026-07-29-final.json new file mode 100644 index 0000000..ca86fa2 --- /dev/null +++ b/benchmarks/results/2026-07-29-final.json @@ -0,0 +1,223 @@ +{ + "schema_version": 1, + "source": { + "baseline_commit": "c0ed162ddf820b8d4ada0127b597cd64a8e35a88", + "optimized_behavior_commit": "1b72146fb52ecf0ab01e4f1b1ff0a6fb05637983", + "rust_version": "1.97.0", + "target": "x86_64-unknown-linux-musl" + }, + "created_utc": "2026-07-29T18:53:03.345430+00:00", + "host": { + "platform": "Linux-6.17.0-1021-azure-x86_64-with-glibc2.39", + "processor_count": 2, + "cpu_set": "0" + }, + "workload": { + "start_seed": 0, + "end_seed": 100, + "seed_count": 100, + "workers": 1, + "writer_sinks": 1, + "channel_size": 64 + }, + "executables": { + "baseline": { + "path": "/tmp/opencode/dsp-optimization/baseline/dsp_seed_finder", + "sha256": "74e96f68daac80a35d4296fd06204d8e4c393785d493c5414ff5679d56f5747c" + }, + "optimized-0.7.0": { + "path": "/tmp/opencode/dsp-optimization/target-final/x86_64-unknown-linux-musl/release/dsp_seed_finder", + "sha256": "b1d3c5d7216848b9fd73cccc36d6e9142574d8ac5d9b0e1e4dbeed4fcc83a31a" + } + }, + "results": [ + { + "sequence": 0, + "variant": "baseline", + "throughput_seeds_per_second": 2.629865, + "wall_seconds": 38.02, + "user_seconds": 28.65, + "system_seconds": 6.75, + "max_rss_kib": 5980, + "minor_page_faults": 3320208, + "major_page_faults": 0, + "voluntary_context_switches": 486, + "involuntary_context_switches": 20167 + }, + { + "sequence": 1, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.669744, + "wall_seconds": 27.25, + "user_seconds": 25.71, + "system_seconds": 0.13, + "max_rss_kib": 5696, + "minor_page_faults": 7036, + "major_page_faults": 0, + "voluntary_context_switches": 377, + "involuntary_context_switches": 7864 + }, + { + "sequence": 2, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.666149, + "wall_seconds": 27.28, + "user_seconds": 25.73, + "system_seconds": 0.1, + "max_rss_kib": 5632, + "minor_page_faults": 7036, + "major_page_faults": 0, + "voluntary_context_switches": 377, + "involuntary_context_switches": 6891 + }, + { + "sequence": 3, + "variant": "baseline", + "throughput_seeds_per_second": 2.648636, + "wall_seconds": 37.75, + "user_seconds": 28.68, + "system_seconds": 6.65, + "max_rss_kib": 5952, + "minor_page_faults": 3320918, + "major_page_faults": 0, + "voluntary_context_switches": 485, + "involuntary_context_switches": 19807 + }, + { + "sequence": 4, + "variant": "baseline", + "throughput_seeds_per_second": 2.648638, + "wall_seconds": 37.75, + "user_seconds": 28.53, + "system_seconds": 6.75, + "max_rss_kib": 5964, + "minor_page_faults": 3320968, + "major_page_faults": 0, + "voluntary_context_switches": 482, + "involuntary_context_switches": 19130 + }, + { + "sequence": 5, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.643891, + "wall_seconds": 27.44, + "user_seconds": 25.64, + "system_seconds": 0.1, + "max_rss_kib": 5676, + "minor_page_faults": 6988, + "major_page_faults": 0, + "voluntary_context_switches": 381, + "involuntary_context_switches": 8187 + }, + { + "sequence": 6, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.68346, + "wall_seconds": 27.15, + "user_seconds": 25.64, + "system_seconds": 0.11, + "max_rss_kib": 5632, + "minor_page_faults": 6860, + "major_page_faults": 0, + "voluntary_context_switches": 380, + "involuntary_context_switches": 7356 + }, + { + "sequence": 7, + "variant": "baseline", + "throughput_seeds_per_second": 2.680183, + "wall_seconds": 37.31, + "user_seconds": 28.66, + "system_seconds": 6.75, + "max_rss_kib": 5976, + "minor_page_faults": 3320980, + "major_page_faults": 0, + "voluntary_context_switches": 478, + "involuntary_context_switches": 24800 + }, + { + "sequence": 8, + "variant": "baseline", + "throughput_seeds_per_second": 2.622841, + "wall_seconds": 38.13, + "user_seconds": 28.66, + "system_seconds": 6.71, + "max_rss_kib": 6008, + "minor_page_faults": 3320999, + "major_page_faults": 0, + "voluntary_context_switches": 489, + "involuntary_context_switches": 22307 + }, + { + "sequence": 9, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.657601, + "wall_seconds": 27.34, + "user_seconds": 25.7, + "system_seconds": 0.12, + "max_rss_kib": 5696, + "minor_page_faults": 7037, + "major_page_faults": 0, + "voluntary_context_switches": 378, + "involuntary_context_switches": 8700 + }, + { + "sequence": 10, + "variant": "optimized-0.7.0", + "throughput_seeds_per_second": 3.694465, + "wall_seconds": 27.07, + "user_seconds": 25.66, + "system_seconds": 0.11, + "max_rss_kib": 5620, + "minor_page_faults": 6970, + "major_page_faults": 0, + "voluntary_context_switches": 379, + "involuntary_context_switches": 8191 + }, + { + "sequence": 11, + "variant": "baseline", + "throughput_seeds_per_second": 2.920451, + "wall_seconds": 34.24, + "user_seconds": 27.5, + "system_seconds": 5.93, + "max_rss_kib": 5920, + "minor_page_faults": 3320262, + "major_page_faults": 0, + "voluntary_context_switches": 449, + "involuntary_context_switches": 6406 + } + ], + "summary": { + "throughput": { + "baseline": { + "median": 2.648637, + "mad": 0.02228399999999975, + "p25": 2.63455775, + "p75": 2.67229675, + "minimum": 2.622841, + "maximum": 2.920451 + }, + "optimized-0.7.0": { + "median": 3.6679465, + "mad": 0.012929500000000038, + "p25": 3.659738, + "p75": 3.680031, + "minimum": 3.643891, + "maximum": 3.694465 + } + }, + "paired_change_percent": { + "median": 37.99626012180245, + "mad": 1.0093619576554183, + "p25": 37.4688720746932, + "p75": 39.193030250875104, + "minimum": 26.50323528797436, + "maximum": 39.54115515435204, + "bootstrap_95_percent_ci": [ + 31.968194086715165, + 39.496515977559426 + ] + } + } +} diff --git a/compose/compose.md b/compose/compose.md index d0647b8..573e46b 100644 --- a/compose/compose.md +++ b/compose/compose.md @@ -1,30 +1,75 @@ -# compose -This folder contains Docker Compose configurations for running project services -individually. - -#### Variations -- **compose.yaml** -Runs only the `seedfinder` container. It is configured to connect to an -individually managed PostgreSQL instance at `host.docker.internal:5432` using -database `dsp`, user `postgres`, and password `rootpassword`. It uses Docker's -built-in `bridge` network so `host-gateway` reaches PostgreSQL's published port -without joining PostgreSQL's Compose network. - -- **compose.postgres.yaml** -Runs PostgreSQL and PostgREST. PostgreSQL is available for individual connections from -local tools or from `seedfinder` in `compose.yaml`. It publishes PostgreSQL on localhost -port `5432`, creates database `dsp`, and persists data in `../data`. PostgREST is -exposed on port `3000` and provides a RESTful API over the `public` schema. - -#### Usage -- Start PostgreSQL + PostgREST: - `docker compose -f compose.postgres.yaml up -d` - -- Start seedfinder only: - `docker compose -f compose.yaml up -d` - -- Connect locally (PostgreSQL): - `postgresql://postgres:rootpassword@localhost:5432/dsp` - -- PostgREST API: - `http://localhost:3000` +# Docker Compose Files + +Run all commands in this document from the repository root. + +## Full Stack + +`compose/compose_full.yaml` uses this startup sequence: + +1. Docker starts PostgreSQL. +2. Docker waits for the PostgreSQL health check to pass. +3. The generator creates the schema. +4. The generator writes the configured seeds. +5. PostgREST starts after the generator exits with code `0`. + +Set the PostgreSQL password: + +```bash +export POSTGRES_PASSWORD='replace-this-password' +``` + +Start the full stack: + +```bash +docker compose -f compose/compose_full.yaml up -d --wait +``` + +The default range contains seeds from 0 to 99. The application programming +interface (API) listens on `127.0.0.1:3000`. + +## PostgreSQL Only + +`compose/compose.postgres.yaml` starts PostgreSQL. PostgreSQL stores data in +`data/`. PostgreSQL listens on `127.0.0.1:5432` by default. + +Set the PostgreSQL password: + +```bash +export POSTGRES_PASSWORD='replace-this-password' +``` + +Start PostgreSQL: + +```bash +docker compose -f compose/compose.postgres.yaml up -d --wait +``` + +Set `POSTGRES_PORT` to use a different host port. + +## Generator Only + +`compose/compose.yaml` starts one finite generator job. The generator connects +to a PostgreSQL server on the Docker host. + +Set the PostgreSQL password: + +```bash +export POSTGRES_PASSWORD='replace-this-password' +``` + +Start the generator: + +```bash +docker compose -f compose/compose.yaml up +``` + +Set `START_SEED` and `END_SEED` before the command to select a range. + +## Stop Services + +```bash +docker compose -f compose/compose_full.yaml down --remove-orphans +``` + +The `docker compose down` command does not remove the bind-mounted database +files in `data/`. diff --git a/docs/performance.md b/docs/performance.md new file mode 100644 index 0000000..830cf8b --- /dev/null +++ b/docs/performance.md @@ -0,0 +1,206 @@ +# Generation Pipeline Performance + +## Purpose + +This document gives the version 0.7.0 performance results, test method, and +recorded optimization experiments. + +Throughput is the primary performance measure. Wall time is elapsed clock time. +The user central processing unit (CPU) time and the system CPU time measure +processor use. +Minor page faults count memory accesses that the operating system resolves +without disk access. Peak resident set size (RSS) measures peak physical memory +use. + +The test did not measure power. The virtual machine does not supply a stable +power counter. The test did not collect hardware performance-counter data. + +## Test System + +| Item | Value | +| --- | --- | +| Date | 2026-07-29 | +| Processor | Intel Xeon Platinum 8370C at 2.80 GHz | +| Available processors | 2 logical processors, 1 physical core | +| Memory | 7.7 GiB | +| Kernel | Linux 6.17.0-1021-azure | +| Docker | 29.6.1 | +| Rust | 1.97.0 | +| Target | `x86_64-unknown-linux-musl` | +| Baseline source | `c0ed162ddf820b8d4ada0127b597cd64a8e35a88` | +| Baseline executable | `74e96f68daac80a35d4296fd06204d8e4c393785d493c5414ff5679d56f5747c` | +| Optimized executable | `b1d3c5d7216848b9fd73cccc36d6e9142574d8ac5d9b0e1e4dbeed4fcc83a31a` | + +The executable values are 256-bit Secure Hash Algorithm (SHA-256) digests. + +## Controlled Method + +The controlled comparison processed seeds from 0 to 99. Each process used one +generator worker and one output sink. The output sink discarded generated +data. The process ran on logical processor 0. The test disabled logs, +checkpoints, and the terminal user interface (TUI). The process did not read +from or write to a database. + +The benchmark measured executable run time only. It did not measure build time. +It ran the executable sequence `A, B, B, A` three times. This sequence gave six +baseline samples and six optimized samples. + +The script calculates the median and median absolute deviation (MAD). The script +uses 20,000 bootstrap samples and bootstrap random seed 0. The script calculates +a 95% confidence interval for the paired change in median throughput. + +Run the comparison with this command: + +```bash +python3 benchmarks/run_comparison.py \ + --baseline /path/to/baseline/dsp_seed_finder \ + --candidate /path/to/optimized/dsp_seed_finder \ + --candidate-name optimized-0.7.0 \ + --output benchmarks/results/local.json \ + --start-seed 0 \ + --end-seed 100 \ + --workers 1 \ + --rounds 3 \ + --cpus 0 +``` + +The script requires GNU `time` and `taskset`. + +## Result + +| Measure | Baseline median | Version 0.7.0 median | Change | +| --- | ---: | ---: | ---: | +| Throughput | 2.649 seeds/s | 3.668 seeds/s | +38.5% | +| Wall time for 100 seeds | 37.750 s | 27.265 s | -27.8% | +| User CPU time | 28.655 s | 25.680 s | -10.4% | +| System CPU time | 6.730 s | 0.110 s | -98.4% | +| Minor page faults | 3,320,943 | 7,012 | -99.8% | +| Peak RSS | 5,970 KiB | 5,654 KiB | -5.3% | + +The paired throughput gain is 38.0%. Its bootstrap 95% interval is 32.0% to +39.5%. The throughput MAD is 0.022 seeds/s for the baseline and 0.013 seeds/s +for version 0.7.0. + +The machine-readable raw result is +[`benchmarks/results/2026-07-29-final.json`](../benchmarks/results/2026-07-29-final.json). + +## Profile Result + +The baseline software profile had these sample percentages: + +- 45.4% to three-dimensional simplex noise. +- 15.6% to page-fault handling. +- 10.6% to terrain-height interpolation. + +The initial 100-seed run caused 3,319,980 minor page faults. The pooled terrain +cache reduced this value to 7,619 in the first cache experiment. + +After the memory changes, 56.2% of samples occurred in simplex noise. A total +of 11.0% of samples occurred in terrain interpolation. + +The virtual machine did not supply cycles, instructions, branches, or cache +misses from `perf`. The test used software sampling and wall measurements. + +## Optimization Decisions + +| Experiment | Decision | Measured result | +| --- | --- | --- | +| Reuse a compact terrain cache for each worker | Keep | +27.14% median throughput. The 95% interval was +16.58% to +31.35%. | +| Compare vein spacing before terrain | Keep | +5.14%. The interval was +2.62% to +8.39%. | +| Store simplex permutations as `u8` | Keep | +2.95%. The interval was +0.91% to +7.56%. | +| Fuse algorithm 12 noise passes | Revert | +0.20%. The interval was -4.76% to +0.75%. | +| Reuse the duplicate noise result in algorithm 9 | Keep | +0.75% median throughput. The 95% interval was -5.15% to +5.00%. | +| Make algorithm 10 turbulence conditional | Revert | -0.92%. The interval was -2.96% to +4.26%. | +| Do not calculate unused detail noise in algorithms 5 and 6 | Keep | +0.38% median throughput. The 95% interval was -1.69% to +7.97%. | +| Reject algorithm 4 craters with partial distance | Revert | -1.84%. The interval was -8.99% to -0.18%. | +| Wait for the first writer item | Keep for reliability | -1.49% median throughput. The 95% interval was -6.72% to +0.87%. | +| Reduce the default writer count and queue capacity | Keep | +0.40% throughput on 500 seeds. | +| Increase the adaptive worker default | Keep | On 500 seeds, 32 workers had 10.6% higher throughput than 8 workers. | + +The machine-readable decision list is +[`benchmarks/results/2026-07-29-experiments.json`](../benchmarks/results/2026-07-29-experiments.json). + +## PostgreSQL Scaling + +PostgreSQL accepted the binary COPY data in the database test. PostgreSQL 16 +and the generator shared the two logical processors. One writer used batches +of 64 seeds. + +| Workers | Seeds | Throughput | Peak generator RSS | +| ---: | ---: | ---: | ---: | +| 2 | 500 | 4.538 seeds/s | 14,432 KiB | +| 8 | 500 | 5.962 seeds/s | 18,008 KiB | +| 16 | 500 | 6.179 seeds/s | 23,876 KiB | +| 32 | 500 | 6.595 seeds/s | 33,808 KiB | + +The generation time is different for each static seed range. More workers +decrease the idle time at the end of the test. Four database writers did not +increase the 32-worker result. One writer is the default. + +The baseline scale test processed 500 seeds. The test failed because the output +sink treated one second without input as a fatal error. Version 0.7.0 waits for +the first item. Version 0.7.0 writes a non-full batch after one second without +input. + +## Correctness And Determinism + +All eight experiments with an output comparison have equal output. Before the +COPY correction, the test compared seeds from 0 to 99. Each file had this +SHA-256 digest: + +```text +3f26eb1b456d5a7b71f40a98ab03e0472b05e41cc31e65740dca002da6afa7d6 +``` + +The COPY correction changed the byte format. The baseline format did not +include field lengths. The baseline also put one header and one trailer in each +seed buffer. PostgreSQL rejects that format as one batch. + +Version 0.7.0 also corrects two moon values. In version 0.7.0, `sun_distance` +and `inside_ds` use the distance from the moon's parent to the star. The +baseline code used the small moon orbital offset. + +PostgreSQL 16 accepted the corrected COPY stream. A ten-seed test inserted 640 +stars and 2,446 planets. The test used three seeds per batch. The last batch +contained fewer than three seeds. + +Two release tests generated seeds from 0 to 999. The files were byte equal. +Each file contained 50,069,759 bytes and had this SHA-256 digest: + +```text +d188c78a555c95f2188db893a8c9890ec2a50848c9b8826f6e72c7104b3b7acd +``` + +The same release executable wrote seeds from 0 to 499 with 8 workers and with +32 workers. Each run inserted 32,000 stars and 122,266 planets. Sorted database +content had these hashes: + +```text +stars: 51ddf9bbe5e3a6dbf17dfa5e65f89621 +planets: 32096b2ffd8476930b437c1b9711cb50 +``` + +The result is valid for one compiled executable. A GNU build and a musl build +gave different planet hashes. Galaxy generation uses trigonometric and power +functions. Different math libraries can round these functions differently. +The release uses the pinned musl executable. + +## No More Changes + +Version 0.7.0 includes the changes marked `Keep` or `Keep for reliability` in +the optimization table. Simplex noise has the largest profile percentage. + +The test did not enable unsafe fast math. Fast math can change terrain +comparisons and random-number consumption. The test did not add hand-written +single instruction, multiple data (SIMD) code. SIMD code can also change +floating-point results. + +The test did not use profile-guided optimization (PGO). PGO can bind code +layout to one training corpus and one hardware profile. + +A dynamic seed queue can reduce range imbalance with fewer workers. A dynamic +seed queue also changes checkpoint rules for seeds that are in progress. +Version 0.7.0 uses more static ranges. + +The tests stopped after the recorded experiments. More low-level changes can +change floating-point results or checkpoint rules. From 4ea2544e5d6c2319513eec469651a92d4c0ba097 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 20:03:43 +0000 Subject: [PATCH 15/18] Disable incomplete logs by default Do not start the unfinished performance logger unless LOG_INTERVAL is set. Create LOG_DIR before an explicitly enabled logger writes its file. The first blank-state Docker test found this defect after ten seconds. The scratch image had no logs directory, and the generator exited before it completed 100 seeds. Release tests pass after the fix. --- inserter/src/main.rs | 6 +++++- inserter/src/misc.rs | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/inserter/src/main.rs b/inserter/src/main.rs index ede1dcd..067b06b 100644 --- a/inserter/src/main.rs +++ b/inserter/src/main.rs @@ -64,7 +64,11 @@ lazy_static! { static ref PG_DBNAME: String = env_str!("PG_DBNAME", "dsp"); static ref CP_INTERVAL: Option = get_env_interval("CHECKPOINT_INTERVAL"); - static ref LOG_INTERVAL: Option = get_env_interval("LOG_INTERVAL"); + static ref LOG_INTERVAL: Option = if std::env::var_os("LOG_INTERVAL").is_some() { + get_env_interval("LOG_INTERVAL") + } else { + None + }; static ref DB_STR: String = { diff --git a/inserter/src/misc.rs b/inserter/src/misc.rs index 341b498..82c2a40 100644 --- a/inserter/src/misc.rs +++ b/inserter/src/misc.rs @@ -193,6 +193,7 @@ pub fn validate_config() -> anyhow::Result<()> { } pub fn write_log() -> anyhow::Result<()> { + std::fs::create_dir_all(&*crate::LOG_DIR)?; let log_path = Path::new(&*crate::LOG_DIR).join(&*crate::LOG_NAME); if !log_path.exists() { From f7f7911723a0d9e152126f3eaaab3b8e00ddedfb Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 20:10:32 +0000 Subject: [PATCH 16/18] Fix standalone database access Use Linux host networking for the standalone generator and connect to PostgreSQL at 127.0.0.1. This keeps the PostgreSQL port bound to host loopback and avoids the inaccessible Docker bridge gateway. The standalone Compose file inserted seeds 100 through 104 into the blank-state database. The API returned seed 100 after the job. Update the Simplified Technical English instructions for this network mode. --- README.md | 4 ++-- compose/compose.md | 4 ++-- compose/compose.yaml | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 0b7c39c..717fe64 100644 --- a/README.md +++ b/README.md @@ -76,8 +76,8 @@ POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ docker compose -f compose/compose.yaml up ``` -The standalone generator connects to PostgreSQL on the Docker host through -`host.docker.internal`. +The standalone generator uses Linux host networking. It connects to PostgreSQL +at `127.0.0.1:5432`. ## Configuration diff --git a/compose/compose.md b/compose/compose.md index 573e46b..b7bf69a 100644 --- a/compose/compose.md +++ b/compose/compose.md @@ -48,8 +48,8 @@ Set `POSTGRES_PORT` to use a different host port. ## Generator Only -`compose/compose.yaml` starts one finite generator job. The generator connects -to a PostgreSQL server on the Docker host. +`compose/compose.yaml` starts one finite generator job. The generator uses Linux +host networking. It connects to PostgreSQL at `127.0.0.1:5432`. Set the PostgreSQL password: diff --git a/compose/compose.yaml b/compose/compose.yaml index 7d847f2..b4a5d2f 100644 --- a/compose/compose.yaml +++ b/compose/compose.yaml @@ -10,9 +10,7 @@ services: PG_PASS: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD} PG_PORT: 5432 PG_DBNAME: dsp - PG_NETLOC: host.docker.internal + PG_NETLOC: 127.0.0.1 LOG_LEVEL: ${LOG_LEVEL:-info} - network_mode: bridge + network_mode: host restart: "no" - extra_hosts: - - "host.docker.internal:host-gateway" From 7d3ebfd722fd424e209496c4354916df5935c73f Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 20:14:05 +0000 Subject: [PATCH 17/18] Pin the published 0.7.0 image --- README.md | 6 ++++++ compose/compose.yaml | 2 +- compose/compose_full.yaml | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 717fe64..1d708fc 100644 --- a/README.md +++ b/README.md @@ -187,6 +187,12 @@ The database files are in `data/`. The release image is `toti330/dsp_seed_finder:0.7.0`. +The published image has this digest: + +```text +sha256:58d968aa4a6661b7b203a6716bd0de7c5a39763d688410b1d600d617513181ca +``` + Examine the remote image: ```bash diff --git a/compose/compose.yaml b/compose/compose.yaml index b4a5d2f..4b6574d 100644 --- a/compose/compose.yaml +++ b/compose/compose.yaml @@ -1,6 +1,6 @@ services: seedfinder: - image: toti330/dsp_seed_finder:0.7.0 + image: toti330/dsp_seed_finder:0.7.0@sha256:58d968aa4a6661b7b203a6716bd0de7c5a39763d688410b1d600d617513181ca container_name: seedfinder environment: START_SEED: ${START_SEED:-0} diff --git a/compose/compose_full.yaml b/compose/compose_full.yaml index a0bc8f4..7bb1031 100644 --- a/compose/compose_full.yaml +++ b/compose/compose_full.yaml @@ -17,7 +17,7 @@ services: timeout: 5s retries: 5 seedfinder: - image: toti330/dsp_seed_finder:0.7.0 + image: toti330/dsp_seed_finder:0.7.0@sha256:58d968aa4a6661b7b203a6716bd0de7c5a39763d688410b1d600d617513181ca container_name: seedfinder environment: START_SEED: ${START_SEED:-0} From 59232b78b5e8065b5697fe6bf6cd253c3a15f9e9 Mon Sep 17 00:00:00 2001 From: toti330 Date: Wed, 29 Jul 2026 20:19:08 +0000 Subject: [PATCH 18/18] Fix the container test command --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1d708fc..0228019 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,8 @@ Run the Rust tests in a container: docker run --rm \ --mount type=bind,source="$PWD/inserter",target=/src,readonly \ --workdir /src \ - rust:1.97.0 \ + --env CARGO_TARGET_DIR=/tmp/target \ + rust:1.97.0@sha256:b92b8c8574f8f3b207fcb0912fb3e2de4041580b5934d90312d53938c9a038a9 \ cargo test --release --locked ```