diff --git a/.github/workflows/benchmark-history.yml b/.github/workflows/benchmark-history.yml new file mode 100644 index 0000000..d4bcb13 --- /dev/null +++ b/.github/workflows/benchmark-history.yml @@ -0,0 +1,205 @@ +name: Benchmark History + +# Measures a small, fixed set of benchmarks once per release, appends the numbers to a committed +# history file, and redraws the chart the README shows. +# +# Two ways in: +# * a published release, which measures that version and adds one point; +# * a manual dispatch listing versions, which measures each of them in ONE job and backfills. +# +# A backfill version is measured as a published package, through BenchmarkAgainstVersion: no tag +# before this workflow carries a benchmark project, so there is no older source to run. A release +# from now on does carry one, and is measured from its own tag through a worktree -- which is also +# what keeps the release path off a race, since the package for a tag is not necessarily on +# nuget.org yet at the moment its release is published. +# +# The backfill running as a single job still matters: separate runs land on different CI hosts, +# and that difference is larger than most releases are. Within one job the points are comparable +# as they stand; across jobs, BaselineBenchmarks is what ties them together. + +on: + release: + types: [published] + workflow_dispatch: + inputs: + versions: + description: "Space-separated released versions to backfill, oldest first" + required: false + # 1.2.2 and 1.2.7 are left out: their Parse throws, which is how every operand here is + # built, so they measure nothing. The backfill would skip them; this saves it the run. + default: "1.3.0 1.4.0 1.4.20 1.4.40 2.0.0 2.0.1" + type: string + +permissions: + contents: write + +concurrency: + group: benchmark-history + cancel-in-progress: false + +env: + DOTNET_VERSION: "10.0" + HISTORY: docs/benchmarks/history.json + CHART: docs/benchmarks/performance.svg + # Already ignored, and ktsu.Sdk regenerates .gitignore on build so a new entry would not last. + RUNS: BenchmarkDotNet.Artifacts + # The set drawn in the README. + HEADLINE_FILTER: >- + *ArithmeticBenchmarks.Add + *ArithmeticBenchmarks.Multiply + *ArithmeticBenchmarks.Divide + *ComparisonBenchmarks.CompareTo + *SignificanceBenchmarks.ReduceToThree + *TextBenchmarks.Parse + *ConversionBenchmarks.FromDouble + # Short runs: three iterations is enough for a trend line, and a release should not tie up a + # runner for half an hour. + BENCHMARK_JOB: short + +jobs: + measure: + name: Measure and chart + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Checkout Repository + uses: actions/checkout@v7 + with: + # The default branch, not the released tag: the results are committed back here, and a + # release event would otherwise leave the checkout detached at the tag, so the push at + # the end would be asking the default branch to move backwards. + ref: ${{ github.event.repository.default_branch }} + fetch-depth: 0 + + - name: Setup .NET SDK ${{ env.DOTNET_VERSION }} + uses: actions/setup-dotnet@v6 + with: + dotnet-version: ${{ env.DOTNET_VERSION }}.x + + # Measured from this checkout, and stamped onto every entry this job produces: everything + # here shares one runner, so one reading of that runner describes all of it. + - name: Measure the reference workload + id: baseline + shell: bash + run: | + set -euo pipefail + dotnet run -c Release --project SignificantNumber.Benchmarks -- \ + --filter '*BaselineBenchmarks.ReferenceWork' \ + --job "$BENCHMARK_JOB" \ + --artifacts "$GITHUB_WORKSPACE/$RUNS/baseline" + ns=$(dotnet run scripts/benchmark-history.cs -- baseline --results "$RUNS/baseline") + echo "Reference workload: $ns ns" + echo "ns=$ns" >> "$GITHUB_OUTPUT" + + - name: Measure the released version + if: github.event_name == 'release' + shell: bash + env: + TAG: ${{ github.event.release.tag_name }} + run: | + set -euo pipefail + version="${TAG#v}" + work="${RUNNER_TEMP}/bench-$version" + git worktree add --detach "$work" "$TAG" + + (cd "$work" && dotnet run -c Release --project SignificantNumber.Benchmarks -- \ + --filter $HEADLINE_FILTER \ + --job "$BENCHMARK_JOB" \ + --artifacts "$GITHUB_WORKSPACE/$RUNS/$version") + + dotnet run scripts/benchmark-history.cs -- ingest \ + --history "$HISTORY" \ + --results "$RUNS/$version" \ + --version "$version" \ + --commit "$(git rev-parse --short "$TAG^{commit}")" \ + --date "$(git log -1 --format=%cs "$TAG")" \ + --run-id "${{ github.run_id }}" \ + --baseline-ns "${{ steps.baseline.outputs.ns }}" + + git worktree remove --force "$work" + + - name: Measure each backfill version + if: github.event_name == 'workflow_dispatch' + shell: bash + env: + VERSIONS: ${{ inputs.versions }} + BASELINE_NS: ${{ steps.baseline.outputs.ns }} + run: | + set -euo pipefail + read -ra versions <<< "$VERSIONS" + + for version in "${versions[@]}"; do + echo "::group::$version" + # Through the environment rather than a -p: switch, because BenchmarkDotNet generates + # and builds a project of its own per run, which a property passed on the command line + # does not reach. MSBuild reads environment variables as properties in every project. + # + # A version whose API the current benchmarks cannot express is reported and skipped, + # rather than failing the whole backfill after the ones before it have been measured. + if ! BenchmarkAgainstVersion="$version" dotnet run -c Release --project SignificantNumber.Benchmarks -- \ + --filter $HEADLINE_FILTER \ + --job "$BENCHMARK_JOB" \ + --artifacts "$GITHUB_WORKSPACE/$RUNS/$version"; then + echo "::warning::$version could not be benchmarked by the current suite; skipping" + echo "::endgroup::" + continue + fi + + tag="v$version" + commit="" + date="" + if git rev-parse -q --verify "$tag^{commit}" >/dev/null; then + commit="$(git rev-parse --short "$tag^{commit}")" + date="$(git log -1 --format=%cs "$tag")" + fi + + # Skipped here too, and for the same reason: a package can build against these + # benchmarks and still throw from every one of them at run time, which BenchmarkDotNet + # reports as a table of NA rather than as a failure. Ingest refuses such a run, and + # the backfill carries on to the next version. + if ! dotnet run scripts/benchmark-history.cs -- ingest \ + --history "$HISTORY" \ + --results "$RUNS/$version" \ + --version "$version" \ + --commit "$commit" \ + --date "$date" \ + --run-id "${{ github.run_id }}" \ + --baseline-ns "$BASELINE_NS"; then + echo "::warning::$version produced no usable measurement; skipping" + fi + echo "::endgroup::" + done + + - name: Redraw the chart + shell: bash + run: dotnet run scripts/benchmark-history.cs -- render --history "$HISTORY" --out "$CHART" + + - name: Commit the history and the chart + shell: bash + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + # Staged first, then compared against the index: on the first run these files are new, + # and `git diff` alone does not see an untracked file, so the run would push nothing and + # still report success. + git add "$HISTORY" "${CHART%.svg}"*.svg + if git diff --cached --quiet; then + echo "Nothing changed." + exit 0 + fi + # [skip ci] so that committing results does not start the pipeline over again. + git commit -m "[bot][skip ci] Update benchmark history" + branch="${{ github.event.repository.default_branch }}" + git pull --rebase origin "$branch" + git push origin "HEAD:$branch" + + - name: Upload the raw reports + if: always() + uses: actions/upload-artifact@v7 + with: + name: benchmark-history-${{ github.run_id }} + path: ${{ env.RUNS }}/ + retention-days: 30 + if-no-files-found: warn diff --git a/Directory.Packages.props b/Directory.Packages.props index ded2c6d..6fd4561 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -4,6 +4,13 @@ + + + + diff --git a/README.md b/README.md index 8ce0b4d..e246a19 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ Upgrading from 1.x? See the [2.0 migration guide](docs/migration-guide-2.0.md). ## Table of contents +- [Performance](#performance) - [Installation](#installation) - [Usage](#usage) - [Creating a SignificantNumber](#creating-a-significantnumber) @@ -44,6 +45,26 @@ Upgrading from 1.x? See the [2.0 migration guide](docs/migration-guide-2.0.md). - [Contributing](#contributing) - [License](#license) +## Performance + + + + Allocated bytes per operation, and time relative to a fixed reference workload, for each SignificantNumber release + + +Every release measures a fixed set of benchmarks and adds a point to the chart; the numbers behind +it are in [`docs/benchmarks/history.json`](docs/benchmarks/history.json), and the suite is +[`SignificantNumber.Benchmarks`](SignificantNumber.Benchmarks/README.md). + +Read the two halves differently. **Allocation is exact** — the same code allocates the same bytes on +any machine, so a step in the top row is always a real change. **Time is measured on shared CI +runners**, where the host a job happens to land on varies more than most releases do, so each time +is divided by a reference workload measured in the same job. That cancels most of the difference +between machines; what is left is indicative rather than precise. + +The step at 2.0 is the type becoming a `readonly record struct` over `PreciseNumber` instead of +deriving from it. A 200-digit addition went from 54,688 bytes and 128 μs to 112 bytes and 340 ns. + ## Installation Install the package with the .NET CLI: diff --git a/SignificantNumber.Benchmarks/ArithmeticBenchmarks.cs b/SignificantNumber.Benchmarks/ArithmeticBenchmarks.cs new file mode 100644 index 0000000..fc69003 --- /dev/null +++ b/SignificantNumber.Benchmarks/ArithmeticBenchmarks.cs @@ -0,0 +1,65 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.SignificantNumber.Benchmarks; + +using BenchmarkDotNet.Attributes; + +/// +/// Measures the arithmetic operators. +/// +/// +/// Every operation here does two things: the arithmetic itself, and then the rounding back to the +/// significance the operands justify. The second half is what separates this library from the +/// number underneath it, and it is why these cost more than the same operation on a +/// PreciseNumber. +/// +[MemoryDiagnoser] +public class ArithmeticBenchmarks +{ + // Assigned in GlobalSetup before anything is measured. Initialised here because this type + // was a class before 2.0, where an unassigned field is a null reference the compiler + // rejects; from 2.0 it is a struct and this is simply its default. + private SignificantNumber left = default!; + private SignificantNumber right = default!; + + /// + /// Gets or sets the number of significant digits in the operands. + /// + [Params(8, 30, 200)] + public int Digits { get; set; } + + /// + /// Prepares the operands. + /// + [GlobalSetup] + public void Setup() + { + left = Operands.Number(Digits, -Digits); + right = Operands.Number(Digits, -Digits, offset: 7); + } + + /// Adds two numbers. + /// The sum. + [Benchmark] + public SignificantNumber Add() => left + right; + + /// Subtracts one number from another. + /// The difference. + [Benchmark] + public SignificantNumber Subtract() => left - right; + + /// Multiplies two numbers. + /// The product. + [Benchmark] + public SignificantNumber Multiply() => left * right; + + /// Divides one number by another. + /// The quotient. + [Benchmark] + public SignificantNumber Divide() => left / right; + + /// Negates a number. + /// The negated number. + [Benchmark] + public SignificantNumber Negate() => -left; +} diff --git a/SignificantNumber.Benchmarks/AssemblyInfo.cs b/SignificantNumber.Benchmarks/AssemblyInfo.cs new file mode 100644 index 0000000..d654f67 --- /dev/null +++ b/SignificantNumber.Benchmarks/AssemblyInfo.cs @@ -0,0 +1,3 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +[assembly: System.Runtime.CompilerServices.InternalsVisibleTo("ktsu.SignificantNumber.Test")] diff --git a/SignificantNumber.Benchmarks/BaselineBenchmarks.cs b/SignificantNumber.Benchmarks/BaselineBenchmarks.cs new file mode 100644 index 0000000..c06e4ba --- /dev/null +++ b/SignificantNumber.Benchmarks/BaselineBenchmarks.cs @@ -0,0 +1,59 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.SignificantNumber.Benchmarks; + +using BenchmarkDotNet.Attributes; + +/// +/// Measures a fixed workload that touches none of this library, so that timings taken on +/// different machines can be compared. +/// +/// +/// Every release is benchmarked in its own CI job, and a job lands on whichever shared runner is +/// free — an x86-64-v3 or v4 host, at whatever clock its neighbours leave it. That difference is +/// routinely larger than the changes a release makes, so a chart of raw times across releases +/// mostly plots the runner. +/// +/// This benchmark is the fixed point that makes the rest comparable. It is integer arithmetic over +/// a value the JIT cannot fold away, chosen because it has no allocation, no library code, and no +/// dependence on anything that changes between versions — so its measured time is a reading of the +/// machine and nothing else. Dividing a benchmark's time by this one's, taken in the same job, +/// cancels most of the difference between hosts. `scripts/benchmark-history.cs` records it on +/// every entry and plots the ratio rather than the nanoseconds. +/// +/// +/// It follows that this method's body must never change. Editing it silently rescales every +/// comparison drawn against history recorded before the edit. +/// +/// +[MemoryDiagnoser] +public class BaselineBenchmarks +{ + // Read from a field rather than written as a literal, so that the loop cannot be constant + // folded into its own answer at JIT time. + private ulong seed; + + /// + /// Sets the starting value. + /// + [GlobalSetup] + public void Setup() => seed = 0xcbf29ce484222325; + + /// + /// Mixes a counter with a multiply-xor-shift step, the way a non-cryptographic hash does. + /// + /// The accumulated value, returned so that nothing here is dead code. + [Benchmark] + public ulong ReferenceWork() + { + ulong accumulator = seed; + + for (int i = 0; i < 256; i++) + { + accumulator = (accumulator ^ (ulong)i) * 0x100000001b3; + accumulator ^= accumulator >> 29; + } + + return accumulator; + } +} diff --git a/SignificantNumber.Benchmarks/BenchmarkConfig.cs b/SignificantNumber.Benchmarks/BenchmarkConfig.cs new file mode 100644 index 0000000..6c166ed --- /dev/null +++ b/SignificantNumber.Benchmarks/BenchmarkConfig.cs @@ -0,0 +1,32 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.SignificantNumber.Benchmarks; + +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Exporters.Json; +using BenchmarkDotNet.Order; + +/// +/// The configuration every benchmark in this assembly runs under. +/// +internal static class BenchmarkConfig +{ + /// + /// Builds the configuration. + /// + /// The configuration to run benchmarks with. + /// + /// Allocation is reported alongside time because most of the cost in this library comes from + /// allocating intermediate values rather than from the arithmetic itself, and a change that + /// trades one for the other should be visible in the same table. Results are kept in + /// declaration order so that a summary reads the way the source does. + /// + internal static IConfig Create() => + ManualConfig.Create(DefaultConfig.Instance) + .AddDiagnoser(MemoryDiagnoser.Default) + .AddColumn(RankColumn.Arabic) + .AddExporter(JsonExporter.Full) + .WithOrderer(new DefaultOrderer(SummaryOrderPolicy.Declared)); +} diff --git a/SignificantNumber.Benchmarks/ComparisonBenchmarks.cs b/SignificantNumber.Benchmarks/ComparisonBenchmarks.cs new file mode 100644 index 0000000..41feec1 --- /dev/null +++ b/SignificantNumber.Benchmarks/ComparisonBenchmarks.cs @@ -0,0 +1,65 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.SignificantNumber.Benchmarks; + +using BenchmarkDotNet.Attributes; + +/// +/// Measures comparing and hashing. +/// +/// +/// Comparison reduces both operands to the significance they share before deciding, so unlike the +/// number underneath it these are not simply a look at two fields. +/// +[MemoryDiagnoser] +public class ComparisonBenchmarks +{ + // Assigned in GlobalSetup before anything is measured. Initialised here because this type + // was a class before 2.0, where an unassigned field is a null reference the compiler + // rejects; from 2.0 it is a struct and this is simply its default. + private SignificantNumber left = default!; + private SignificantNumber right = default!; + private SignificantNumber same = default!; + + /// + /// Gets or sets the number of significant digits in the operands. + /// + [Params(8, 30, 200)] + public int Digits { get; set; } + + /// + /// Prepares the operands. + /// + [GlobalSetup] + public void Setup() + { + left = Operands.Number(Digits, -Digits); + right = Operands.Number(Digits, -Digits, offset: 7); + same = Operands.Number(Digits, -Digits); + } + + /// Compares two equal numbers. + /// Whether they are equal. + [Benchmark] + public bool EqualsSame() => left == same; + + /// Compares two different numbers. + /// Whether they are equal. + [Benchmark] + public bool EqualsDifferent() => left == right; + + /// Orders two numbers. + /// Whether the left is smaller. + [Benchmark] + public bool LessThan() => left < right; + + /// Orders two numbers, returning the sign of their difference. + /// The comparison result. + [Benchmark] + public int CompareTo() => left.CompareTo(right); + + /// Hashes a number. + /// The hash code. + [Benchmark] + public int GetHashCodeBenchmark() => left.GetHashCode(); +} diff --git a/SignificantNumber.Benchmarks/ConversionBenchmarks.cs b/SignificantNumber.Benchmarks/ConversionBenchmarks.cs new file mode 100644 index 0000000..ff24e5a --- /dev/null +++ b/SignificantNumber.Benchmarks/ConversionBenchmarks.cs @@ -0,0 +1,51 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.SignificantNumber.Benchmarks; + +using BenchmarkDotNet.Attributes; + +/// +/// Measures conversion to and from the primitive numeric types. +/// +/// +/// These are the boundary a caller crosses to get a value in and an answer out, so they are on the +/// path of anything that uses the library at all rather than only of code doing arithmetic in it. +/// +[MemoryDiagnoser] +public class ConversionBenchmarks +{ + // Read from fields rather than written as literals, so that the JIT cannot fold a conversion + // into its own answer at compile time. + private double doubleValue; + private int intValue; + // Assigned in GlobalSetup before anything is measured. Initialised here because this type + // was a class before 2.0, where an unassigned field is a null reference the compiler + // rejects; from 2.0 it is a struct and this is simply its default. + private SignificantNumber number = default!; + + /// + /// Prepares the operands. + /// + [GlobalSetup] + public void Setup() + { + doubleValue = 123.456789; + intValue = 123456; + number = Operands.Number(30, -15); + } + + /// Converts a double in. + /// The converted number. + [Benchmark] + public SignificantNumber FromDouble() => doubleValue.ToSignificantNumber(); + + /// Converts an int in. + /// The converted number. + [Benchmark] + public SignificantNumber FromInt32() => intValue.ToSignificantNumber(); + + /// Converts out to a double. + /// The converted value. + [Benchmark] + public double ToDouble() => double.CreateTruncating(number); +} diff --git a/SignificantNumber.Benchmarks/Operands.cs b/SignificantNumber.Benchmarks/Operands.cs new file mode 100644 index 0000000..e4d56bb --- /dev/null +++ b/SignificantNumber.Benchmarks/Operands.cs @@ -0,0 +1,71 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.SignificantNumber.Benchmarks; + +using System.Globalization; + +/// +/// Builds the operands the benchmarks run against. +/// +/// +/// Values are derived from a fixed digit pattern rather than a random source so that two runs of +/// the same benchmark, on the same machine or on different ones, are measuring the same work. +/// +/// Everything is built by parsing text, because that is the only construction route the published +/// packages share. A benchmark that reached for an internal factory could measure the working copy +/// and nothing before it, which would leave the release chart with a single point. +/// +/// +internal static class Operands +{ + /// + /// An arbitrary but fixed run of non-repeating digits to slice operands out of. + /// + private const string DigitPattern = + "31415926535897932384626433832795028841971693993751" + + "05820974944592307816406286208998628034825342117067" + + "98214808651328230664709384460955058223172535940812" + + "84811174502841027019385211055596446229489549303819"; + + /// + /// Renders decimal text with exactly significant digits. + /// + /// The number of significant digits. + /// The power of ten to scale by. + /// Shifts the window into the digit pattern, so that two operands of the + /// same length are not identical. + /// The decimal text, in scientific notation. + internal static string Text(int digits, int exponent, int offset = 0) + { + char[] characters = new char[digits]; + for (int i = 0; i < digits; i++) + { + characters[i] = DigitPattern[(i + offset) % DigitPattern.Length]; + } + + // A leading or trailing zero would make the value's digit count differ from what was asked + // for, because significance is counted from the first non-zero digit and trailing zeros are + // stripped. + if (characters[0] == '0') + { + characters[0] = '4'; + } + + if (characters[digits - 1] == '0') + { + characters[digits - 1] = '7'; + } + + return string.Create(CultureInfo.InvariantCulture, $"{new string(characters)}E{exponent}"); + } + + /// + /// Builds a number with the given significant digit count and exponent. + /// + /// The number of significant digits. + /// The power of ten to scale by. + /// Shifts the window into the digit pattern. + /// The constructed number. + internal static SignificantNumber Number(int digits, int exponent, int offset = 0) => + SignificantNumber.Parse(Text(digits, exponent, offset), CultureInfo.InvariantCulture); +} diff --git a/SignificantNumber.Benchmarks/Program.cs b/SignificantNumber.Benchmarks/Program.cs new file mode 100644 index 0000000..54401be --- /dev/null +++ b/SignificantNumber.Benchmarks/Program.cs @@ -0,0 +1,18 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.SignificantNumber.Benchmarks; + +using BenchmarkDotNet.Running; + +/// +/// Entry point for the benchmark suite. +/// +internal static class Program +{ + /// + /// Runs the benchmarks named on the command line, or prompts for a selection when none are. + /// + /// Command line arguments, forwarded to BenchmarkDotNet. + internal static void Main(string[] args) => + _ = BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, BenchmarkConfig.Create()); +} diff --git a/SignificantNumber.Benchmarks/README.md b/SignificantNumber.Benchmarks/README.md new file mode 100644 index 0000000..cddb0d4 --- /dev/null +++ b/SignificantNumber.Benchmarks/README.md @@ -0,0 +1,82 @@ +# SignificantNumber Benchmarks + +A [BenchmarkDotNet](https://benchmarkdotnet.org) suite covering the operations that dominate real +use of `SignificantNumber`: arithmetic, comparison, reducing significance, parsing, and conversion +from the primitive numeric types. + +## Running + +From the repository root: + +```bash +# Pick benchmarks from an interactive list +dotnet run -c Release --project SignificantNumber.Benchmarks + +# Run everything +dotnet run -c Release --project SignificantNumber.Benchmarks -- --filter '*' + +# Run one class, or one method +dotnet run -c Release --project SignificantNumber.Benchmarks -- --filter '*ArithmeticBenchmarks*' +dotnet run -c Release --project SignificantNumber.Benchmarks -- --filter '*.Divide' +``` + +## Measuring a published release + +Set `BenchmarkAgainstVersion` and the suite measures that package instead of the working copy: + +```bash +BenchmarkAgainstVersion=1.4.40 dotnet run -c Release --project SignificantNumber.Benchmarks -- --filter '*.Add' +``` + +Set it **in the environment, not with `-p:`**. BenchmarkDotNet generates and builds a project of its +own for each run, and a property passed on the command line does not reach that project — it would +build the benchmark assembly against the version you asked for and the harness against the one +pinned centrally, which fails to compile if the type changed shape between them. MSBuild reads +environment variables as properties in every project, so the environment form reaches both. + +This switch is how `docs/benchmarks/` is filled. No tag in this repository carries a benchmark +project, so there is no older source to check out and run; and measuring packages is the better +comparison anyway, because every version is timed by identical benchmark code rather than by +whatever each tag happened to ship. + +### What the older packages cannot be asked + +The history starts at 1.3.0, and one panel of it starts at 1.4.20. + +`Operands` builds every value by parsing text, because that is the only construction route all the +published versions share. 1.2.2 and 1.2.7 throw `NotSupportedException` from `Parse`, so they +compile against these benchmarks, run, and report a table of `NA`. Ingest refuses a run with no +measurement in it rather than putting a release on the axis with nothing under it, and the backfill +reports it and moves on. + +Significance reduction is spelled three ways across the versions: 1.2.x has no precision overload of +`ToSignificantNumber` at all, and 1.3 and 1.4.0 hang it off the `PreciseNumber` base, where the +receiver's type argument has to be written out. Carrying three spellings of one benchmark would +measure the spellings, so `SignificanceBenchmarks.cs` is left out of builds against anything older +than 1.4.20 and the chart draws that panel with a gap. Everything else in those versions is still +measured. + +### Why nothing here touches an internal member + +Measuring published packages is also why nothing here reaches for an internal. The +`InternalsVisibleTo` that would expose one is not in the packages already published, so a benchmark +built on internals could only ever measure the working copy. + +It is why there is no formatting benchmark, too. Before 2.0 this type derived from `PreciseNumber` +rather than wrapping it, so `ToString` was an inherited member: measuring it would mean referencing +that package here, and the reference makes `Parse` and `GetHashCode` ambiguous against their +inherited counterparts. One operation is not worth the whole release history before 2.0. + +## Reading the results + +Most classes are parameterised by `Digits` (8, 30, 200). That axis is the point: significance is +carried in a `BigInteger`, so anything touching digits one at a time looks fine at 8 and collapses +at 200. Read across the `Digits` column, not down one value of it. + +Allocation is reported alongside time and matters just as much. Between 1.4.40 and 2.0.1 a 200-digit +`Add` went from 54,688 bytes to 112, and from 128 μs to 340 ns — the type became a value type and +stopped allocating an object per intermediate. + +Every operation pays twice: the arithmetic, and then the rounding back to the significance the +operands justify. `SignificanceBenchmarks` measures that second half on its own, so it sets a floor +under everything in `ArithmeticBenchmarks`. diff --git a/SignificantNumber.Benchmarks/SignificanceBenchmarks.cs b/SignificantNumber.Benchmarks/SignificanceBenchmarks.cs new file mode 100644 index 0000000..2fbac3c --- /dev/null +++ b/SignificantNumber.Benchmarks/SignificanceBenchmarks.cs @@ -0,0 +1,44 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.SignificantNumber.Benchmarks; + +using BenchmarkDotNet.Attributes; + +/// +/// Measures reducing a number to a stated number of significant digits. +/// +/// +/// This is the operation the library is named for, and the one every arithmetic result pays for +/// internally, so its cost sets a floor under everything in . +/// The work tracks how many digits are being discarded rather than how many are kept. +/// +[MemoryDiagnoser] +public class SignificanceBenchmarks +{ + // Assigned in GlobalSetup before anything is measured. Initialised here because this type + // was a class before 2.0, where an unassigned field is a null reference the compiler + // rejects; from 2.0 it is a struct and this is simply its default. + private SignificantNumber number = default!; + + /// + /// Gets or sets the number of significant digits in the operand. + /// + [Params(8, 30, 200)] + public int Digits { get; set; } + + /// + /// Prepares the operand. + /// + [GlobalSetup] + public void Setup() => number = Operands.Number(Digits, -Digits); + + /// Reduces to three significant digits, discarding most of them. + /// The reduced number. + [Benchmark] + public SignificantNumber ReduceToThree() => number.ToSignificantNumber(3); + + /// Reduces to half the digits it has, so the work scales with the operand. + /// The reduced number. + [Benchmark] + public SignificantNumber ReduceToHalf() => number.ToSignificantNumber(Digits / 2); +} diff --git a/SignificantNumber.Benchmarks/SignificantNumber.Benchmarks.csproj b/SignificantNumber.Benchmarks/SignificantNumber.Benchmarks.csproj new file mode 100644 index 0000000..4db8945 --- /dev/null +++ b/SignificantNumber.Benchmarks/SignificantNumber.Benchmarks.csproj @@ -0,0 +1,60 @@ + + + + + + Exe + net10.0 + + + SignificantNumber.Benchmarks + ktsu.SignificantNumber.Benchmarks + + true + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SignificantNumber.Benchmarks/TextBenchmarks.cs b/SignificantNumber.Benchmarks/TextBenchmarks.cs new file mode 100644 index 0000000..7292485 --- /dev/null +++ b/SignificantNumber.Benchmarks/TextBenchmarks.cs @@ -0,0 +1,46 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.SignificantNumber.Benchmarks; + +using System.Globalization; + +using BenchmarkDotNet.Attributes; + +/// +/// Measures parsing. +/// +/// +/// Parsing is how every operand in this suite is built, so its cost is worth knowing on its own +/// rather than only as part of a setup step. +/// +/// Formatting is deliberately absent. Before 2.0 this type derived from PreciseNumber and +/// inherited its ToString, so measuring it would mean referencing that package here, and +/// the reference makes Parse and GetHashCode ambiguous against the inherited +/// members — which would cost the whole release history before 2.0 to measure one operation. +/// +/// +[MemoryDiagnoser] +public class TextBenchmarks +{ + // Left to default rather than initialised from a named constant: the constant resolves + // through the transitive PreciseNumber package, which this project deliberately does not + // reference, and GlobalSetup assigns both before anything is measured. + private string text = ""; + + /// + /// Gets or sets the number of significant digits in the operand. + /// + [Params(8, 30, 200)] + public int Digits { get; set; } + + /// + /// Prepares the text. + /// + [GlobalSetup] + public void Setup() => text = Operands.Text(Digits, -Digits); + + /// Parses decimal text in scientific notation. + /// The parsed number. + [Benchmark] + public SignificantNumber Parse() => SignificantNumber.Parse(text, CultureInfo.InvariantCulture); +} diff --git a/SignificantNumber.sln b/SignificantNumber.sln index ae4145e..a44479b 100644 --- a/SignificantNumber.sln +++ b/SignificantNumber.sln @@ -7,6 +7,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SignificantNumber", "Signif EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SignificantNumber.Test", "SignificantNumber.Test\SignificantNumber.Test.csproj", "{96F1AF95-952D-4BFA-8489-DEAB052E4BC7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SignificantNumber.Benchmarks", "SignificantNumber.Benchmarks\SignificantNumber.Benchmarks.csproj", "{5B1C7A24-9E63-4D08-A7F1-2C4D6E8B3A05}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -21,6 +23,10 @@ Global {96F1AF95-952D-4BFA-8489-DEAB052E4BC7}.Debug|Any CPU.Build.0 = Debug|Any CPU {96F1AF95-952D-4BFA-8489-DEAB052E4BC7}.Release|Any CPU.ActiveCfg = Release|Any CPU {96F1AF95-952D-4BFA-8489-DEAB052E4BC7}.Release|Any CPU.Build.0 = Release|Any CPU + {5B1C7A24-9E63-4D08-A7F1-2C4D6E8B3A05}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5B1C7A24-9E63-4D08-A7F1-2C4D6E8B3A05}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5B1C7A24-9E63-4D08-A7F1-2C4D6E8B3A05}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5B1C7A24-9E63-4D08-A7F1-2C4D6E8B3A05}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/docs/benchmarks/history.json b/docs/benchmarks/history.json new file mode 100644 index 0000000..f255be7 --- /dev/null +++ b/docs/benchmarks/history.json @@ -0,0 +1,583 @@ +{ + "schemaVersion": 1, + "entries": [ + { + "version": "1.3.0", + "commit": "67809e0", + "date": "2025-04-13", + "cpu": "Intel Xeon Processor 2.80GHz", + "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", + "baselineNs": 452.8087, + "runId": "local-seed", + "benchmarks": { + "ArithmeticBenchmarks.Add": { + "Digits=8": { + "meanNs": 620.4483, + "allocatedBytes": 80 + }, + "Digits=30": { + "meanNs": 5533.9367, + "allocatedBytes": 3080 + }, + "Digits=200": { + "meanNs": 126712.8964, + "allocatedBytes": 54608 + } + }, + "ArithmeticBenchmarks.Divide": { + "Digits=8": { + "meanNs": 4530.0272, + "allocatedBytes": 2322 + }, + "Digits=30": { + "meanNs": 8833.5941, + "allocatedBytes": 5524 + }, + "Digits=200": { + "meanNs": 131684.1884, + "allocatedBytes": 57182 + } + }, + "ArithmeticBenchmarks.Multiply": { + "Digits=8": { + "meanNs": 1412.3979, + "allocatedBytes": 424 + }, + "Digits=30": { + "meanNs": 11288.7724, + "allocatedBytes": 6824 + }, + "Digits=200": { + "meanNs": 268323.1639, + "allocatedBytes": 126248 + } + }, + "ComparisonBenchmarks.CompareTo": { + "Digits=8": { + "meanNs": 580.2629, + "allocatedBytes": 80 + }, + "Digits=30": { + "meanNs": 4996.2886, + "allocatedBytes": 3040 + }, + "Digits=200": { + "meanNs": 125983.5019, + "allocatedBytes": 54592 + } + }, + "ConversionBenchmarks.FromDouble": { + "": { + "meanNs": 1705.2617, + "allocatedBytes": 1185 + } + }, + "TextBenchmarks.Parse": { + "Digits=8": { + "meanNs": 534.2614, + "allocatedBytes": 80 + }, + "Digits=30": { + "meanNs": 4905.4913, + "allocatedBytes": 3040 + }, + "Digits=200": { + "meanNs": 89796.4008, + "allocatedBytes": 54384 + } + } + } + }, + { + "version": "1.4.0", + "commit": "7ba23ac", + "date": "2025-04-13", + "cpu": "Intel Xeon Processor 2.80GHz", + "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", + "baselineNs": 452.8087, + "runId": "local-seed", + "benchmarks": { + "ArithmeticBenchmarks.Add": { + "Digits=8": { + "meanNs": 628.3597, + "allocatedBytes": 80 + }, + "Digits=30": { + "meanNs": 5371.4127, + "allocatedBytes": 3080 + }, + "Digits=200": { + "meanNs": 129460.3076, + "allocatedBytes": 54608 + } + }, + "ArithmeticBenchmarks.Divide": { + "Digits=8": { + "meanNs": 4391.0955, + "allocatedBytes": 2322 + }, + "Digits=30": { + "meanNs": 8843.35, + "allocatedBytes": 5524 + }, + "Digits=200": { + "meanNs": 133177.2462, + "allocatedBytes": 57182 + } + }, + "ArithmeticBenchmarks.Multiply": { + "Digits=8": { + "meanNs": 1397.8879, + "allocatedBytes": 424 + }, + "Digits=30": { + "meanNs": 11067.6285, + "allocatedBytes": 6824 + }, + "Digits=200": { + "meanNs": 269427.1546, + "allocatedBytes": 126248 + } + }, + "ComparisonBenchmarks.CompareTo": { + "Digits=8": { + "meanNs": 583.7263, + "allocatedBytes": 80 + }, + "Digits=30": { + "meanNs": 5159.6631, + "allocatedBytes": 3040 + }, + "Digits=200": { + "meanNs": 127106.1182, + "allocatedBytes": 54592 + } + }, + "ConversionBenchmarks.FromDouble": { + "": { + "meanNs": 1688.6434, + "allocatedBytes": 1185 + } + }, + "TextBenchmarks.Parse": { + "Digits=8": { + "meanNs": 513.3549, + "allocatedBytes": 80 + }, + "Digits=30": { + "meanNs": 4828.2697, + "allocatedBytes": 3040 + }, + "Digits=200": { + "meanNs": 86648.0418, + "allocatedBytes": 54384 + } + } + } + }, + { + "version": "1.4.20", + "commit": "8f6c917", + "date": "2026-07-17", + "cpu": "Intel Xeon Processor 2.80GHz", + "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", + "baselineNs": 452.8087, + "runId": "local-seed", + "benchmarks": { + "ArithmeticBenchmarks.Add": { + "Digits=8": { + "meanNs": 619.5243, + "allocatedBytes": 160 + }, + "Digits=30": { + "meanNs": 5229.8272, + "allocatedBytes": 3160 + }, + "Digits=200": { + "meanNs": 128075.502, + "allocatedBytes": 54688 + } + }, + "ArithmeticBenchmarks.Divide": { + "Digits=8": { + "meanNs": 4254.1535, + "allocatedBytes": 2298 + }, + "Digits=30": { + "meanNs": 8760.2066, + "allocatedBytes": 5500 + }, + "Digits=200": { + "meanNs": 138068.185, + "allocatedBytes": 57158 + } + }, + "ArithmeticBenchmarks.Multiply": { + "Digits=8": { + "meanNs": 1406.8425, + "allocatedBytes": 424 + }, + "Digits=30": { + "meanNs": 11165.3128, + "allocatedBytes": 6824 + }, + "Digits=200": { + "meanNs": 271426.4561, + "allocatedBytes": 126248 + } + }, + "ComparisonBenchmarks.CompareTo": { + "Digits=8": { + "meanNs": 606.5947, + "allocatedBytes": 160 + }, + "Digits=30": { + "meanNs": 5412.4328, + "allocatedBytes": 3200 + }, + "Digits=200": { + "meanNs": 128128.2382, + "allocatedBytes": 54752 + } + }, + "ConversionBenchmarks.FromDouble": { + "": { + "meanNs": 1680.0394, + "allocatedBytes": 1161 + } + }, + "SignificanceBenchmarks.ReduceToThree": { + "Digits=8": { + "meanNs": 301.3614, + "allocatedBytes": 80 + }, + "Digits=30": { + "meanNs": 1668.0534, + "allocatedBytes": 1440 + }, + "Digits=200": { + "meanNs": 17373.0794, + "allocatedBytes": 26944 + } + }, + "TextBenchmarks.Parse": { + "Digits=8": { + "meanNs": 518.674, + "allocatedBytes": 80 + }, + "Digits=30": { + "meanNs": 4660.0772, + "allocatedBytes": 3040 + }, + "Digits=200": { + "meanNs": 85208.2083, + "allocatedBytes": 54384 + } + } + } + }, + { + "version": "1.4.40", + "commit": "247a814", + "date": "2026-09-14", + "cpu": "Intel Xeon Processor 2.80GHz", + "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", + "baselineNs": 452.8087, + "runId": "local-seed", + "benchmarks": { + "ArithmeticBenchmarks.Add": { + "Digits=8": { + "meanNs": 629.4364, + "allocatedBytes": 160 + }, + "Digits=30": { + "meanNs": 5265.3493, + "allocatedBytes": 3160 + }, + "Digits=200": { + "meanNs": 129051.806, + "allocatedBytes": 54688 + } + }, + "ArithmeticBenchmarks.Divide": { + "Digits=8": { + "meanNs": 4367.9198, + "allocatedBytes": 2298 + }, + "Digits=30": { + "meanNs": 9094.7489, + "allocatedBytes": 5500 + }, + "Digits=200": { + "meanNs": 132643.3647, + "allocatedBytes": 57158 + } + }, + "ArithmeticBenchmarks.Multiply": { + "Digits=8": { + "meanNs": 1414.4, + "allocatedBytes": 424 + }, + "Digits=30": { + "meanNs": 11243.8029, + "allocatedBytes": 6824 + }, + "Digits=200": { + "meanNs": 270934.7518, + "allocatedBytes": 126248 + } + }, + "ComparisonBenchmarks.CompareTo": { + "Digits=8": { + "meanNs": 610.7298, + "allocatedBytes": 160 + }, + "Digits=30": { + "meanNs": 5261.6772, + "allocatedBytes": 3200 + }, + "Digits=200": { + "meanNs": 128266.5607, + "allocatedBytes": 54752 + } + }, + "ConversionBenchmarks.FromDouble": { + "": { + "meanNs": 1636.9918, + "allocatedBytes": 1161 + } + }, + "SignificanceBenchmarks.ReduceToThree": { + "Digits=8": { + "meanNs": 292.872, + "allocatedBytes": 80 + }, + "Digits=30": { + "meanNs": 1685.0539, + "allocatedBytes": 1440 + }, + "Digits=200": { + "meanNs": 17588.527, + "allocatedBytes": 26944 + } + }, + "TextBenchmarks.Parse": { + "Digits=8": { + "meanNs": 509.4118, + "allocatedBytes": 80 + }, + "Digits=30": { + "meanNs": 4703.075, + "allocatedBytes": 3040 + }, + "Digits=200": { + "meanNs": 87108.5511, + "allocatedBytes": 54384 + } + } + } + }, + { + "version": "2.0.0", + "commit": "10f1426", + "date": "2026-09-15", + "cpu": "Intel Xeon Processor 2.80GHz", + "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", + "baselineNs": 452.8087, + "runId": "local-seed", + "benchmarks": { + "ArithmeticBenchmarks.Add": { + "Digits=8": { + "meanNs": 35.6765, + "allocatedBytes": 0 + }, + "Digits=30": { + "meanNs": 118.0456, + "allocatedBytes": 40 + }, + "Digits=200": { + "meanNs": 336.1947, + "allocatedBytes": 112 + } + }, + "ArithmeticBenchmarks.Divide": { + "Digits=8": { + "meanNs": 613.5962, + "allocatedBytes": 248 + }, + "Digits=30": { + "meanNs": 938.4486, + "allocatedBytes": 312 + }, + "Digits=200": { + "meanNs": 3370.5546, + "allocatedBytes": 528 + } + }, + "ArithmeticBenchmarks.Multiply": { + "Digits=8": { + "meanNs": 155.5987, + "allocatedBytes": 32 + }, + "Digits=30": { + "meanNs": 755.4034, + "allocatedBytes": 248 + }, + "Digits=200": { + "meanNs": 2920.5572, + "allocatedBytes": 528 + } + }, + "ComparisonBenchmarks.CompareTo": { + "Digits=8": { + "meanNs": 9.2312, + "allocatedBytes": 0 + }, + "Digits=30": { + "meanNs": 11.4472, + "allocatedBytes": 0 + }, + "Digits=200": { + "meanNs": 11.9296, + "allocatedBytes": 0 + } + }, + "ConversionBenchmarks.FromDouble": { + "": { + "meanNs": 359.4532, + "allocatedBytes": 0 + } + }, + "SignificanceBenchmarks.ReduceToThree": { + "Digits=8": { + "meanNs": 61.8836, + "allocatedBytes": 0 + }, + "Digits=30": { + "meanNs": 157.9816, + "allocatedBytes": 80 + }, + "Digits=200": { + "meanNs": 211.8753, + "allocatedBytes": 224 + } + }, + "TextBenchmarks.Parse": { + "Digits=8": { + "meanNs": 195.492, + "allocatedBytes": 0 + }, + "Digits=30": { + "meanNs": 490.0903, + "allocatedBytes": 40 + }, + "Digits=200": { + "meanNs": 3057.2608, + "allocatedBytes": 112 + } + } + } + }, + { + "version": "2.0.1", + "commit": "563cca0", + "date": "2026-09-16", + "cpu": "Intel Xeon Processor 2.80GHz", + "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", + "baselineNs": 452.8087, + "runId": "local-seed", + "benchmarks": { + "ArithmeticBenchmarks.Add": { + "Digits=8": { + "meanNs": 35.6456, + "allocatedBytes": 0 + }, + "Digits=30": { + "meanNs": 123.9326, + "allocatedBytes": 40 + }, + "Digits=200": { + "meanNs": 345.8313, + "allocatedBytes": 112 + } + }, + "ArithmeticBenchmarks.Divide": { + "Digits=8": { + "meanNs": 623.6429, + "allocatedBytes": 248 + }, + "Digits=30": { + "meanNs": 943.3598, + "allocatedBytes": 312 + }, + "Digits=200": { + "meanNs": 3441.8766, + "allocatedBytes": 528 + } + }, + "ArithmeticBenchmarks.Multiply": { + "Digits=8": { + "meanNs": 159.6749, + "allocatedBytes": 32 + }, + "Digits=30": { + "meanNs": 769.6604, + "allocatedBytes": 248 + }, + "Digits=200": { + "meanNs": 2986.04, + "allocatedBytes": 528 + } + }, + "ComparisonBenchmarks.CompareTo": { + "Digits=8": { + "meanNs": 8.7894, + "allocatedBytes": 0 + }, + "Digits=30": { + "meanNs": 11.7078, + "allocatedBytes": 0 + }, + "Digits=200": { + "meanNs": 12.4922, + "allocatedBytes": 0 + } + }, + "ConversionBenchmarks.FromDouble": { + "": { + "meanNs": 369.7793, + "allocatedBytes": 0 + } + }, + "SignificanceBenchmarks.ReduceToThree": { + "Digits=8": { + "meanNs": 62.995, + "allocatedBytes": 0 + }, + "Digits=30": { + "meanNs": 164.9354, + "allocatedBytes": 80 + }, + "Digits=200": { + "meanNs": 204.6833, + "allocatedBytes": 224 + } + }, + "TextBenchmarks.Parse": { + "Digits=8": { + "meanNs": 186.0136, + "allocatedBytes": 0 + }, + "Digits=30": { + "meanNs": 485.5936, + "allocatedBytes": 40 + }, + "Digits=200": { + "meanNs": 2372.9787, + "allocatedBytes": 112 + } + } + } + } + ] +} diff --git a/docs/benchmarks/performance-dark.svg b/docs/benchmarks/performance-dark.svg new file mode 100644 index 0000000..453eb63 --- /dev/null +++ b/docs/benchmarks/performance-dark.svg @@ -0,0 +1,173 @@ + + + +SignificantNumber performance by release +6 releases · newest 2.0.1 · 2026-09-16 + +Allocated bytes per operation +Deterministic: the same code allocates the same bytes on any machine. +Add (30 digits) + + + + + + + + +40 B +3.0 KB +Multiply (30 digits) + + + + + + + + +248 B +6.7 KB +Divide (30 digits) + + + + + + + + +312 B +5.4 KB +CompareTo (30 digits) + + + + + + + + +0 B +3.0 KB +Reduce to 3 (30 digits) + + + + + + +80 B +1.4 KB +Parse (30 digits) + + + + + + + + +40 B +3.0 KB +From double + + + + + + + + +0 B +1.2 KB + +Time, as a multiple of a fixed reference workload +Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster. +Add (30 digits) + + + + + + + + +0.274× +12.2× +Multiply (30 digits) + + + + + + + + +1.70× +24.9× +Divide (30 digits) + + + + + + + + +2.08× +19.5× +CompareTo (30 digits) + + + + + + + + +0.0259× +11.0× +Reduce to 3 (30 digits) + + + + + + +0.364× +3.68× +Parse (30 digits) + + + + + + + + +1.07× +10.8× +From double + + + + + + + + +0.817× +3.77× +releases, oldest to newest: 1.3.0 → 1.4.0 → 1.4.20 → 1.4.40 → 2.0.0 → 2.0.1 +Measured on Intel Xeon Processor 2.80GHz. Full tables: SignificantNumber.Benchmarks. + diff --git a/docs/benchmarks/performance.svg b/docs/benchmarks/performance.svg new file mode 100644 index 0000000..660e489 --- /dev/null +++ b/docs/benchmarks/performance.svg @@ -0,0 +1,173 @@ + + + +SignificantNumber performance by release +6 releases · newest 2.0.1 · 2026-09-16 + +Allocated bytes per operation +Deterministic: the same code allocates the same bytes on any machine. +Add (30 digits) + + + + + + + + +40 B +3.0 KB +Multiply (30 digits) + + + + + + + + +248 B +6.7 KB +Divide (30 digits) + + + + + + + + +312 B +5.4 KB +CompareTo (30 digits) + + + + + + + + +0 B +3.0 KB +Reduce to 3 (30 digits) + + + + + + +80 B +1.4 KB +Parse (30 digits) + + + + + + + + +40 B +3.0 KB +From double + + + + + + + + +0 B +1.2 KB + +Time, as a multiple of a fixed reference workload +Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster. +Add (30 digits) + + + + + + + + +0.274× +12.2× +Multiply (30 digits) + + + + + + + + +1.70× +24.9× +Divide (30 digits) + + + + + + + + +2.08× +19.5× +CompareTo (30 digits) + + + + + + + + +0.0259× +11.0× +Reduce to 3 (30 digits) + + + + + + +0.364× +3.68× +Parse (30 digits) + + + + + + + + +1.07× +10.8× +From double + + + + + + + + +0.817× +3.77× +releases, oldest to newest: 1.3.0 → 1.4.0 → 1.4.20 → 1.4.40 → 2.0.0 → 2.0.1 +Measured on Intel Xeon Processor 2.80GHz. Full tables: SignificantNumber.Benchmarks. + diff --git a/scripts/benchmark-history.cs b/scripts/benchmark-history.cs new file mode 100644 index 0000000..34e88a1 --- /dev/null +++ b/scripts/benchmark-history.cs @@ -0,0 +1,614 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +// Accumulates benchmark results per release and draws them for the README. +// +// dotnet run scripts/benchmark-history.cs -- ingest --history --results --version +// dotnet run scripts/benchmark-history.cs -- render --history --out +// +// A file-based app rather than a project: it is tooling, it is the same language as the library, +// and the SDK that builds the library already runs it with nothing else installed. The work still +// lives in a class rather than in top-level statements, so the analyzers judge each method. + +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.RegularExpressions; + +return BenchmarkHistory.Run(args); + +/// Reads BenchmarkDotNet reports into a per-release history, and draws it. +internal static partial class BenchmarkHistory +{ + private const int SchemaVersion = 1; + private const string BaselineKey = "BaselineBenchmarks.ReferenceWork"; + private const int Columns = 4; + private const int CellWidth = 228; + private const int CellHeight = 132; + private const int Left = 56; + + /// The benchmarks the README draws, in order. + /// + /// Everything measured is stored; this only decides what the picture shows, so it can change + /// without re-running anything. + /// + private static readonly (string Key, string? Parameters, string Label)[] Headline = + [ + ("ArithmeticBenchmarks.Add", "30", "Add"), + ("ArithmeticBenchmarks.Multiply", "30", "Multiply"), + ("ArithmeticBenchmarks.Divide", "30", "Divide"), + ("ComparisonBenchmarks.CompareTo", "30", "CompareTo"), + ("SignificanceBenchmarks.ReduceToThree", "30", "Reduce to 3"), + ("TextBenchmarks.Parse", "30", "Parse"), + ("ConversionBenchmarks.FromDouble", null, "From double"), + ]; + + /// + /// Validated for colour-vision separation against both surfaces: every check passes, worst + /// adjacent pair dE 24.7 light and 26.8 dark. + /// + private static readonly Dictionary Themes = new(StringComparer.Ordinal) + { + ["light"] = new("#fcfcfb", "#0b0b0b", "#52514e", "#e4e3df", "#2a78d6", "#eb6834"), + ["dark"] = new("#1a1a19", "#ffffff", "#c3c2b7", "#333330", "#3987e5", "#d95926"), + }; + + private sealed record Theme( + string Surface, string Ink, string Muted, string Grid, string Alloc, string Time); + + internal static int Run(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine("Expected 'ingest', 'render', or 'baseline'."); + return 2; + } + + Dictionary options = ReadOptions(args.Skip(1)); + try + { + return args[0] switch + { + "ingest" => Ingest(options), + "render" => Render(options), + "baseline" => PrintBaseline(options), + _ => Unknown(args[0]), + }; + } + catch (InvalidOperationException problem) + { + Console.Error.WriteLine(problem.Message); + return 2; + } + } + + /// Prints the reference workload's mean, for a workflow to carry between steps. + private static int PrintBaseline(Dictionary options) + { + string directory = Required(options, "results"); + string[] reports = Directory.GetFiles(directory, "*-report-full.json", SearchOption.AllDirectories); + if (reports.Length == 0) + { + Console.Error.WriteLine($"No *-report-full.json under {directory}"); + return 1; + } + + (var measured, _, _) = ReadReports(reports); + double? baseline = Baseline(measured, ""); + if (baseline is null) + { + Console.Error.WriteLine($"No {BaselineKey} measurement under {directory}"); + return 1; + } + + Console.WriteLine(baseline.Value.ToString(CultureInfo.InvariantCulture)); + return 0; + } + + private static int Unknown(string command) + { + Console.Error.WriteLine($"Unknown command '{command}'."); + return 2; + } + + private static Dictionary ReadOptions(IEnumerable rest) + { + Dictionary found = new(StringComparer.Ordinal); + string? name = null; + foreach (string argument in rest) + { + if (argument.StartsWith("--", StringComparison.Ordinal)) + { + name = argument[2..]; + found[name] = ""; + } + else if (name is not null) + { + found[name] = argument; + name = null; + } + } + + return found; + } + + private static string Required(Dictionary options, string name) => + options.TryGetValue(name, out string? value) && value.Length > 0 + ? value + : throw new InvalidOperationException($"--{name} is required"); + + private static string Optional(Dictionary options, string name, string fallback = "") => + options.TryGetValue(name, out string? value) && value.Length > 0 ? value : fallback; + + private static int Ingest(Dictionary options) + { + string resultsDirectory = Required(options, "results"); + string[] reports = Directory.GetFiles(resultsDirectory, "*-report-full.json", SearchOption.AllDirectories); + Array.Sort(reports, StringComparer.Ordinal); + if (reports.Length == 0) + { + Console.Error.WriteLine($"No *-report-full.json under {resultsDirectory}"); + return 1; + } + + (var measured, string cpu, string runtime) = ReadReports(reports); + double? baseline = Baseline(measured, Optional(options, "baseline-ns")); + if (baseline is null) + { + Console.Error.WriteLine( + $"warning: no {BaselineKey} measurement and no --baseline-ns; " + + "this entry's times will not be comparable across runners"); + } + + string version = Required(options, "version"); + JsonObject benchmarks = Benchmarks(measured); + if (benchmarks.Count == 0) + { + // Reports with every row reading NA: the harness built and ran, and each benchmark + // threw. An older package whose Parse is a NotSupportedException does exactly this. + // Recording it would put a release on the axis with nothing under it, which reads as + // a release that was measured and found to cost nothing. + Console.Error.WriteLine( + $"No benchmark in {resultsDirectory} produced a measurement; {version} not recorded"); + return 1; + } + + JsonObject record = new() + { + ["version"] = version, + ["commit"] = Optional(options, "commit"), + ["date"] = Optional(options, "date", DateTime.UtcNow.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)), + ["cpu"] = cpu, + ["runtime"] = runtime, + ["baselineNs"] = baseline, + ["runId"] = Optional(options, "run-id"), + ["benchmarks"] = benchmarks, + }; + + string historyPath = Required(options, "history"); + JsonObject history = LoadHistory(historyPath); + JsonArray entries = history["entries"]!.AsArray(); + + // A version is measured once. Re-running a release replaces its entry rather than doubling it. + for (int index = entries.Count - 1; index >= 0; index--) + { + if (string.Equals(entries[index]?["version"]?.GetValue(), version, StringComparison.Ordinal)) + { + entries.RemoveAt(index); + } + } + + entries.Add((JsonNode?)record); + Reorder(entries); + + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(historyPath))!); + File.WriteAllText(historyPath, history.ToJsonString(new JsonSerializerOptions { WriteIndented = true }) + "\n"); + + Console.WriteLine( + $"ingested {version}: {record["benchmarks"]!.AsObject().Count} benchmarks, " + + $"baseline {baseline?.ToString(CultureInfo.InvariantCulture) ?? "none"} ns, " + + $"cpu {(cpu.Length > 0 ? cpu : "unknown")}"); + return 0; + } + + private static (SortedDictionary> Measured, string Cpu, string Runtime) + ReadReports(string[] reports) + { + SortedDictionary> measured = new(StringComparer.Ordinal); + string cpu = ""; + string runtime = ""; + + foreach (JsonNode document in reports.Select(report => JsonNode.Parse(File.ReadAllText(report))!)) + { + if (cpu.Length == 0 && document["HostEnvironmentInfo"] is JsonNode environment) + { + cpu = (environment["ProcessorName"]?.GetValue() ?? "").Trim(); + runtime = (environment["RuntimeVersion"]?.GetValue() ?? "").Trim(); + } + + foreach (JsonNode? entry in document["Benchmarks"]?.AsArray() ?? []) + { + if (entry?["Statistics"]?["Mean"] is not JsonNode mean) + { + continue; + } + + string key = BenchmarkKey(entry["FullName"]?.GetValue() ?? ""); + string parameters = (entry["Parameters"]?.GetValue() ?? "").Trim(); + if (!measured.TryGetValue(key, out List? cases)) + { + cases = []; + measured[key] = cases; + } + + Measurement measurement = new( + Math.Round(mean.GetValue(), 4), + entry["Memory"]?["BytesAllocatedPerOperation"]?.GetValue() ?? 0); + int existing = cases.FindIndex(one => string.Equals(one.Parameters, parameters, StringComparison.Ordinal)); + if (existing >= 0) + { + cases[existing] = new(parameters, measurement); + } + else + { + cases.Add(new(parameters, measurement)); + } + } + } + + return (measured, cpu, runtime); + } + + private sealed record Measurement(double MeanNs, long AllocatedBytes); + + private sealed record ParameterCase(string Parameters, Measurement Value); + + private static double? Baseline( + SortedDictionary> measured, string given) + { + if (measured.TryGetValue(BaselineKey, out List? cases) && cases.Count > 0) + { + return cases[0].Value.MeanNs; + } + + return double.TryParse(given, NumberStyles.Float, CultureInfo.InvariantCulture, out double parsed) + ? parsed + : null; + } + + private static JsonObject Benchmarks( + SortedDictionary> measured) + { + JsonObject benchmarks = []; + foreach ((string key, List cases) in measured) + { + if (string.Equals(key, BaselineKey, StringComparison.Ordinal)) + { + continue; + } + + JsonObject byParameters = []; + foreach (ParameterCase one in cases) + { + byParameters[one.Parameters] = new JsonObject + { + ["meanNs"] = one.Value.MeanNs, + ["allocatedBytes"] = one.Value.AllocatedBytes, + }; + } + + benchmarks[key] = byParameters; + } + + return benchmarks; + } + + private static void Reorder(JsonArray entries) + { + JsonNode[] ordered = + [ + .. entries + .Select(node => node!.DeepClone()) + .OrderBy(node => node["version"]?.GetValue() ?? "", VersionOrder.Instance), + ]; + + entries.Clear(); + foreach (JsonNode node in ordered) + { + entries.Add((JsonNode?)node); + } + } + + private static string BenchmarkKey(string fullName) + { + string bare = fullName.Split('(')[0]; + string[] parts = bare.Split('.'); + return parts.Length >= 2 ? $"{parts[^2]}.{parts[^1]}" : bare; + } + + private static JsonObject LoadHistory(string path) + { + if (!File.Exists(path)) + { + return new JsonObject { ["schemaVersion"] = SchemaVersion, ["entries"] = new JsonArray() }; + } + + JsonObject history = JsonNode.Parse(File.ReadAllText(path))!.AsObject(); + history["schemaVersion"] ??= SchemaVersion; + history["entries"] ??= new JsonArray(); + return history; + } + + /// Orders versions numerically, keeping anything unparseable first in name order. + private sealed class VersionOrder : IComparer + { + internal static readonly VersionOrder Instance = new(); + + public int Compare(string? left, string? right) + { + int[] first = Numbers(left); + int[] second = Numbers(right); + for (int index = 0; index < Math.Min(first.Length, second.Length); index++) + { + if (first[index] != second[index]) + { + return first[index].CompareTo(second[index]); + } + } + + return first.Length != second.Length + ? first.Length.CompareTo(second.Length) + : string.CompareOrdinal(left, right); + } + + private static int[] Numbers(string? text) => + [.. DigitRun().Matches(text ?? "").Select(match => int.Parse(match.Value, CultureInfo.InvariantCulture))]; + } + + [GeneratedRegex("[0-9]+")] + private static partial Regex DigitRun(); + + private static int Render(Dictionary options) + { + string historyPath = Required(options, "history"); + JsonArray entries = LoadHistory(historyPath)["entries"]!.AsArray(); + if (entries.Count == 0) + { + Console.Error.WriteLine($"{historyPath} has no entries to draw"); + return 1; + } + + string output = Required(options, "out"); + string extension = Path.GetExtension(output); + string stem = output[..^extension.Length]; + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(output))!); + + List written = []; + foreach (string name in (string[])["light", "dark"]) + { + string path = string.Equals(name, "light", StringComparison.Ordinal) + ? output + : $"{stem}-dark{extension}"; + File.WriteAllText(path, Draw(entries, Themes[name])); + written.Add(path); + } + + Console.WriteLine($"rendered {entries.Count} releases to {string.Join(", ", written)}"); + return 0; + } + + private static string Draw(JsonArray entries, Theme theme) + { + string[] labels = [.. entries.Select(entry => entry!["version"]?.GetValue() ?? "?")]; + int width = Left + (Columns * CellWidth) + 24; + int rows = (Headline.Length + Columns - 1) / Columns; + int height = 72 + (((34 + (rows * CellHeight)) * 2) + 54); + + StringBuilder svg = new(); + Preamble(svg, theme, width, height, entries); + + int y = 72; + foreach (bool isTime in (bool[])[false, true]) + { + Section(svg, theme, entries, labels.Length, y, isTime); + y += 34 + (rows * CellHeight); + } + + Footer(svg, entries, labels, y - 4); + svg.AppendLine(""); + return svg.ToString(); + } + + private static void Preamble(StringBuilder svg, Theme theme, int width, int height, JsonArray entries) + { + svg.AppendLine(CultureInfo.InvariantCulture, $""""""); + svg.AppendLine(""); + svg.AppendLine(CultureInfo.InvariantCulture, $""""""); + svg.AppendLine(CultureInfo.InvariantCulture, $"""SignificantNumber performance by release"""); + + JsonNode latest = entries[^1]!; + string date = latest["date"]?.GetValue() ?? ""; + string suffix = date.Length > 0 ? " · " + Escape(date) : ""; + svg.AppendLine(CultureInfo.InvariantCulture, $"""{entries.Count} releases · newest {Escape(latest["version"]?.GetValue() ?? "?")}{suffix}"""); + } + + private static void Section(StringBuilder svg, Theme theme, JsonArray entries, int points, int y, bool isTime) + { + string colour = isTime ? theme.Time : theme.Alloc; + string title = isTime + ? "Time, as a multiple of a fixed reference workload" + : "Allocated bytes per operation"; + string note = isTime + ? "Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster." + : "Deterministic: the same code allocates the same bytes on any machine."; + + svg.AppendLine(CultureInfo.InvariantCulture, $""""""); + svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(title)}"""); + svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(note)}"""); + + for (int position = 0; position < Headline.Length; position++) + { + (string key, string? parameters, string label) = Headline[position]; + double?[] values = [.. entries.Select(entry => Value(entry!, key, parameters, isTime))]; + Panel( + svg, + Left + (position % Columns * CellWidth), + y + 26 + (position / Columns * CellHeight), + label + (parameters is null ? "" : $" ({parameters} digits)"), + points, + values, + isTime, + colour, + theme); + } + } + + private static double? Value(JsonNode entry, string key, string? parameters, bool isTime) + { + if (entry["benchmarks"]?[key] is not JsonObject cases || cases.Count == 0) + { + return null; + } + + JsonNode? measurement = parameters is null + ? cases.First().Value + : cases.FirstOrDefault(pair => pair.Key.Contains(parameters, StringComparison.Ordinal)).Value; + if (measurement is null) + { + return null; + } + + if (!isTime) + { + return measurement["allocatedBytes"]!.GetValue(); + } + + double? baseline = entry["baselineNs"]?.GetValue(); + return baseline is > 0 ? measurement["meanNs"]!.GetValue() / baseline : null; + } + + private static void Footer(StringBuilder svg, JsonArray entries, string[] labels, int axisY) + { + List ticks = []; + for (int index = 0; index < labels.Length; index++) + { + // Every label while they fit. Thinning them reads as the whole list, which would say + // there were fewer releases than there were. + if (labels.Length > 12 && index > 0 && index < labels.Length - 1 && index % 2 == 1) + { + continue; + } + + ticks.Add(Escape(labels[index])); + } + + svg.AppendLine(CultureInfo.InvariantCulture, $"""releases, oldest to newest: {string.Join(" → ", ticks)}"""); + + string[] cpus = + [ + .. entries + .Select(entry => entry!["cpu"]?.GetValue() ?? "") + .Where(name => name.Length > 0) + .Distinct(StringComparer.Ordinal) + .OrderBy(name => name, StringComparer.Ordinal), + ]; + string measured = cpus.Length > 0 ? string.Join(", ", cpus) : "an unrecorded CPU"; + svg.AppendLine(CultureInfo.InvariantCulture, $"""Measured on {Escape(measured)}. Full tables: SignificantNumber.Benchmarks."""); + } + + /// One small multiple: a single series, so colour carries no identity of its own. + private static void Panel( + StringBuilder svg, int x0, int y0, string title, int points, double?[] values, + bool isTime, string colour, Theme theme) + { + double plotTop = y0 + 22; + double plotBottom = y0 + CellHeight - 12 - 20; + double plotLeft = x0 + 6; + double plotRight = x0 + CellWidth - 16 - 10; + + svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(title)}"""); + + (int Index, double Value)[] present = + [ + .. values + .Select((value, index) => (Index: index, Value: value)) + .Where(point => point.Value.HasValue) + .Select(point => (point.Index, point.Value!.Value)), + ]; + + if (present.Length == 0) + { + svg.AppendLine(CultureInfo.InvariantCulture, $"""not measured"""); + return; + } + + // Zero-based: these are magnitudes, and a clipped axis would exaggerate every wobble. + double highest = present.Max(point => point.Value); + double top = highest > 0 ? highest * 1.25 : 1.0; + + double X(int index) => points == 1 + ? (plotLeft + plotRight) / 2 + : plotLeft + ((plotRight - plotLeft) * index / (points - 1)); + double Y(double value) => plotBottom - ((plotBottom - plotTop) * (value / top)); + + svg.AppendLine(CultureInfo.InvariantCulture, $""""""); + + if (present.Length > 1) + { + string line = string.Join(" ", present.Select(point => $"{F(X(point.Index))},{F(Y(point.Value))}")); + svg.AppendLine(CultureInfo.InvariantCulture, $""""""); + } + + foreach ((int index, double value) in present) + { + // A 2px surface ring keeps markers legible where the line passes behind them. + svg.AppendLine(CultureInfo.InvariantCulture, $""""""); + } + + (int lastIndex, double lastValue) = present[^1]; + string anchor = lastIndex == points - 1 ? "end" : "middle"; + svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(Label(lastValue, isTime))}"""); + + (int firstIndex, double firstValue) = present[0]; + if (firstIndex != lastIndex) + { + svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(Label(firstValue, isTime))}"""); + } + } + + /// One decimal is ample for an SVG coordinate, and keeps the committed diff small. + private static string F(double value) => value.ToString("0.0", CultureInfo.InvariantCulture); + + private static string Label(double value, bool isTime) => + isTime ? RatioLabel(value) : ByteLabel(value); + + private static string ByteLabel(double value) => + value <= 0 ? "0 B" + : value >= 1024 ? (value / 1024).ToString("0.0", CultureInfo.InvariantCulture) + " KB" + : value.ToString("0", CultureInfo.InvariantCulture) + " B"; + + /// Three significant figures, so a 0.0331x and a 15.3x are both legible. + private static string RatioLabel(double value) => + value <= 0 ? "0×" + : value >= 100 ? value.ToString("0", CultureInfo.InvariantCulture) + "×" + : value >= 10 ? value.ToString("0.0", CultureInfo.InvariantCulture) + "×" + : value >= 1 ? value.ToString("0.00", CultureInfo.InvariantCulture) + "×" + : value >= 0.1 ? value.ToString("0.000", CultureInfo.InvariantCulture) + "×" + : value.ToString("0.0000", CultureInfo.InvariantCulture) + "×"; + + private static string Escape(string text) => + text + .Replace("&", "&", StringComparison.Ordinal) + .Replace("<", "<", StringComparison.Ordinal) + .Replace(">", ">", StringComparison.Ordinal) + .Replace("\"", """, StringComparison.Ordinal); +}