diff --git a/.github/workflows/benchmark-history.yml b/.github/workflows/benchmark-history.yml index d4bcb13..39b7b25 100644 --- a/.github/workflows/benchmark-history.yml +++ b/.github/workflows/benchmark-history.yml @@ -52,6 +52,7 @@ env: *SignificanceBenchmarks.ReduceToThree *TextBenchmarks.Parse *ConversionBenchmarks.FromDouble + *AbstractionCostBenchmarks.* # Short runs: three iterations is enough for a trend line, and a release should not tie up a # runner for half an hour. BENCHMARK_JOB: short diff --git a/SignificantNumber.Benchmarks/AbstractionCostBenchmarks.cs b/SignificantNumber.Benchmarks/AbstractionCostBenchmarks.cs new file mode 100644 index 0000000..ef69643 --- /dev/null +++ b/SignificantNumber.Benchmarks/AbstractionCostBenchmarks.cs @@ -0,0 +1,162 @@ +// Copyright (c) 2023-2026 ktsu-dev contributors + +namespace ktsu.SignificantNumber.Benchmarks; + +using System.Globalization; + +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; + +/// +/// Measures what this type costs against the same arithmetic on a bare . +/// +/// +/// +/// Every other class here answers "how long does this operation take", which is only readable next +/// to something. This one supplies the something: the primitive a caller would otherwise have +/// used. The same class, with the same loops and the same methodology, is in ktsu.PreciseNumber and +/// ktsu.Semantics, so the three answers are comparable with each other as well as with +/// . Against ktsu.PreciseNumber in particular the difference is what +/// significance tracking adds, since this type is built on that one. +/// +/// +/// The bare method is the BenchmarkDotNet baseline, so the answer is the Ratio column rather +/// than two rows divided by hand. A ratio here is not expected to be 1.00 and is not a defect when +/// it is not: every operation pays twice, once for the arbitrary-precision arithmetic and again for +/// rounding the result back to the significance the operands justify, and both are costs paid for +/// something a cannot do at all. What the number is for is watching that cost +/// across releases. +/// +/// +/// Why these are loops. A single operation over operands that do not change is +/// loop-invariant, and the JIT hoists it out of the measurement entirely — for +/// that leaves a method indistinguishable from an empty one, and a ratio +/// against an empty method means nothing. Here each iteration feeds the next, so there is nothing +/// to hoist and both sides are measurable. +/// +/// +/// Which way the loop biases the answer. Both sides pay the same counter and branch, and it +/// is a dependency chain, so most of that overlaps the arithmetic; whatever does not is added +/// equally to numerator and denominator and pulls the ratio toward 1.00. A ratio here is therefore +/// a floor on the real cost rather than the whole of it. +/// +/// +/// Why the operands stay bounded, and why they are short. The number underneath carries as +/// many digits as the arithmetic produces, so a chain that compounded its operand would measure +/// that growth rather than the operation; both loops accumulate instead. The operands are also +/// chosen to be values a can hold, so the two sides are doing the same +/// arithmetic on the same numbers rather than being handed different problems. How the cost grows +/// with digits is a different question, and answers it across +/// its Digits axis. +/// +/// +[MemoryDiagnoser] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[CategoriesColumn] +public class AbstractionCostBenchmarks +{ + /// + /// Operations per invocation. Enough that the loop's own cost is a small share of the work, + /// few enough that the arbitrary-precision side still finishes an iteration promptly. + /// + private const int Operations = 256; + + private const string SeedText = "1234.5678901234"; + private const string StepText = "0.0009765625"; + private const string OtherText = "3.14159265358979"; + + private double bareSeed; + private double bareStep; + private double bareOther; + + // Assigned in GlobalSetup before anything is measured. Initialised here because this type + // was a class before 2.0, where an unassigned field is a null reference the compiler + // rejects; from 2.0 it is a struct and this is simply its default. The backfill measures + // those releases too, so the file has to compile against both shapes. + private SignificantNumber significantSeed = default!; + private SignificantNumber significantStep = default!; + private SignificantNumber significantOther = default!; + + /// + /// Prepares the operands, parsed from the same text on both sides. + /// + [GlobalSetup] + public void Setup() + { + bareSeed = double.Parse(SeedText, CultureInfo.InvariantCulture); + bareStep = double.Parse(StepText, CultureInfo.InvariantCulture); + bareOther = double.Parse(OtherText, CultureInfo.InvariantCulture); + + significantSeed = SignificantNumber.Parse(SeedText, CultureInfo.InvariantCulture); + significantStep = SignificantNumber.Parse(StepText, CultureInfo.InvariantCulture); + significantOther = SignificantNumber.Parse(OtherText, CultureInfo.InvariantCulture); + } + + /// Adds along a chain, on a bare double. + /// The accumulated value. + [BenchmarkCategory("Add")] + [Benchmark(Baseline = true, OperationsPerInvoke = Operations)] + public double BareAdd() + { + double accumulator = bareSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += bareStep; + } + + return accumulator; + } + + /// Adds along the same chain, on this type. + /// The accumulated value. + [BenchmarkCategory("Add")] + [Benchmark(OperationsPerInvoke = Operations)] + public SignificantNumber SignificantAdd() + { + SignificantNumber accumulator = significantSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += significantStep; + } + + return accumulator; + } + + /// Multiplies and accumulates, on a bare double. + /// The accumulated value. + [BenchmarkCategory("Multiply")] + [Benchmark(Baseline = true, OperationsPerInvoke = Operations)] + public double BareMultiply() + { + double accumulator = 0d; + double value = bareSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += value * bareOther; + value += bareStep; + } + + return accumulator; + } + + /// Multiplies and accumulates over the same values, on this type. + /// The accumulated value. + [BenchmarkCategory("Multiply")] + [Benchmark(OperationsPerInvoke = Operations)] + public SignificantNumber SignificantMultiply() + { + SignificantNumber accumulator = SignificantNumber.Zero; + SignificantNumber value = significantSeed; + + for (int i = 0; i < Operations; i++) + { + accumulator += value * significantOther; + value += significantStep; + } + + return accumulator; + } +} diff --git a/SignificantNumber.Benchmarks/README.md b/SignificantNumber.Benchmarks/README.md index cddb0d4..5ce9edd 100644 --- a/SignificantNumber.Benchmarks/README.md +++ b/SignificantNumber.Benchmarks/README.md @@ -67,6 +67,44 @@ rather than wrapping it, so `ToString` was an inherited member: measuring it wou that package here, and the reference makes `Parse` and `GetHashCode` ambiguous against their inherited counterparts. One operation is not worth the whole release history before 2.0. +## What this type costs against a bare double + +`AbstractionCostBenchmarks` is the one benchmark here whose answer is a ratio rather than a +duration. Every other class says how long an operation takes, which is only readable beside +something; this supplies the something — the primitive a caller would otherwise have used. + +The same class, with the same loops and the same methodology, is in `ktsu.PreciseNumber` and +`ktsu.Semantics`. Against PreciseNumber in particular the difference is what significance tracking +adds, since this type is built on that one. + +| release | `Add` | `Multiply` | +|---|---|---| +| 1.3.0 | 1377.7× | 5809.6× | +| 1.4.40 | 1352.4× | 5944.6× | +| 2.0.0 | **89.7×** | **550.5×** | +| 2.0.1 | 89.9× | 569.3× | + +Becoming a value type in 2.0 took roughly **15× off add and 10× off multiply**. For comparison, +the same change in `ktsu.PreciseNumber` underneath moved its ratios by about 15% — so most of what +2.0 recovered here was this layer's own allocation, not the number beneath it. + +**The ratio is not expected to be 1 and is not a defect for being large.** Every operation pays +twice: once for the arbitrary-precision arithmetic and again for rounding the result back to the +significance the operands justify, and both buy something a `double` cannot do at all. What the +chart's third section is for is noticing the day it moves. + +Three things decide how the number should be read: + +- **These are loops.** A single operation over operands that do not change is loop-invariant and + the JIT hoists it out, which would leave the `double` side indistinguishable from an empty method + and the ratio meaningless. Each iteration feeds the next, so there is nothing to hoist. +- **The loop's own cost biases toward 1**, being paid by both sides, so a ratio is a floor on the + real cost rather than the whole of it. +- **Both loops accumulate rather than compound**, because the number underneath carries as many + digits as the arithmetic produces and a compounding chain would measure that growth instead of + the operation. How the cost grows with digits is a different question, and `ArithmeticBenchmarks` + answers it across the `Digits` axis. + ## Reading the results Most classes are parameterised by `Digits` (8, 30, 200). That axis is the point: significance is diff --git a/docs/benchmarks/history.json b/docs/benchmarks/history.json index f255be7..96e8900 100644 --- a/docs/benchmarks/history.json +++ b/docs/benchmarks/history.json @@ -7,82 +7,106 @@ "date": "2025-04-13", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 452.8087, + "baselineNs": 456.0134, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9948, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3907, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.SignificantAdd": { + "": { + "meanNs": 1370.5137, + "allocatedBytes": 512 + } + }, + "AbstractionCostBenchmarks.SignificantMultiply": { + "": { + "meanNs": 8079.4789, + "allocatedBytes": 3577 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 620.4483, + "meanNs": 609.0306, "allocatedBytes": 80 }, "Digits=30": { - "meanNs": 5533.9367, + "meanNs": 5435.117, "allocatedBytes": 3080 }, "Digits=200": { - "meanNs": 126712.8964, + "meanNs": 132400.8595, "allocatedBytes": 54608 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 4530.0272, + "meanNs": 4398.7718, "allocatedBytes": 2322 }, "Digits=30": { - "meanNs": 8833.5941, + "meanNs": 9397.4201, "allocatedBytes": 5524 }, "Digits=200": { - "meanNs": 131684.1884, + "meanNs": 136876.1867, "allocatedBytes": 57182 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 1412.3979, + "meanNs": 1437.835, "allocatedBytes": 424 }, "Digits=30": { - "meanNs": 11288.7724, + "meanNs": 11783.5388, "allocatedBytes": 6824 }, "Digits=200": { - "meanNs": 268323.1639, + "meanNs": 281441.0424, "allocatedBytes": 126248 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 580.2629, + "meanNs": 590.7034, "allocatedBytes": 80 }, "Digits=30": { - "meanNs": 4996.2886, + "meanNs": 5548.5213, "allocatedBytes": 3040 }, "Digits=200": { - "meanNs": 125983.5019, + "meanNs": 130040.8798, "allocatedBytes": 54592 } }, "ConversionBenchmarks.FromDouble": { "": { - "meanNs": 1705.2617, + "meanNs": 1742.2447, "allocatedBytes": 1185 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 534.2614, + "meanNs": 538.952, "allocatedBytes": 80 }, "Digits=30": { - "meanNs": 4905.4913, + "meanNs": 4749.1447, "allocatedBytes": 3040 }, "Digits=200": { - "meanNs": 89796.4008, + "meanNs": 89236.7277, "allocatedBytes": 54384 } } @@ -94,82 +118,106 @@ "date": "2025-04-13", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 452.8087, + "baselineNs": 456.0134, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9882, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.4719, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.SignificantAdd": { + "": { + "meanNs": 1376.9484, + "allocatedBytes": 512 + } + }, + "AbstractionCostBenchmarks.SignificantMultiply": { + "": { + "meanNs": 8026.9203, + "allocatedBytes": 3497 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 628.3597, + "meanNs": 623.6477, "allocatedBytes": 80 }, "Digits=30": { - "meanNs": 5371.4127, + "meanNs": 5372.7202, "allocatedBytes": 3080 }, "Digits=200": { - "meanNs": 129460.3076, + "meanNs": 132484.5563, "allocatedBytes": 54608 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 4391.0955, + "meanNs": 4553.3087, "allocatedBytes": 2322 }, "Digits=30": { - "meanNs": 8843.35, + "meanNs": 9805.5876, "allocatedBytes": 5524 }, "Digits=200": { - "meanNs": 133177.2462, + "meanNs": 136795.9536, "allocatedBytes": 57182 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 1397.8879, + "meanNs": 1434.5849, "allocatedBytes": 424 }, "Digits=30": { - "meanNs": 11067.6285, + "meanNs": 11579.2372, "allocatedBytes": 6824 }, "Digits=200": { - "meanNs": 269427.1546, + "meanNs": 279464.2951, "allocatedBytes": 126248 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 583.7263, + "meanNs": 595.6474, "allocatedBytes": 80 }, "Digits=30": { - "meanNs": 5159.6631, + "meanNs": 5237.6692, "allocatedBytes": 3040 }, "Digits=200": { - "meanNs": 127106.1182, + "meanNs": 131068.1743, "allocatedBytes": 54592 } }, "ConversionBenchmarks.FromDouble": { "": { - "meanNs": 1688.6434, + "meanNs": 1648.4436, "allocatedBytes": 1185 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 513.3549, + "meanNs": 521.8459, "allocatedBytes": 80 }, "Digits=30": { - "meanNs": 4828.2697, + "meanNs": 4738.0426, "allocatedBytes": 3040 }, "Digits=200": { - "meanNs": 86648.0418, + "meanNs": 89652.2228, "allocatedBytes": 54384 } } @@ -181,96 +229,120 @@ "date": "2026-07-17", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 452.8087, + "baselineNs": 456.0134, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9892, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3715, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.SignificantAdd": { + "": { + "meanNs": 1331.6091, + "allocatedBytes": 512 + } + }, + "AbstractionCostBenchmarks.SignificantMultiply": { + "": { + "meanNs": 8145.8635, + "allocatedBytes": 3577 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 619.5243, + "meanNs": 623.8432, "allocatedBytes": 160 }, "Digits=30": { - "meanNs": 5229.8272, + "meanNs": 5362.3294, "allocatedBytes": 3160 }, "Digits=200": { - "meanNs": 128075.502, + "meanNs": 130916.2509, "allocatedBytes": 54688 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 4254.1535, + "meanNs": 4323.6373, "allocatedBytes": 2298 }, "Digits=30": { - "meanNs": 8760.2066, + "meanNs": 9347.8901, "allocatedBytes": 5500 }, "Digits=200": { - "meanNs": 138068.185, + "meanNs": 135586.3996, "allocatedBytes": 57158 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 1406.8425, + "meanNs": 1435.9347, "allocatedBytes": 424 }, "Digits=30": { - "meanNs": 11165.3128, + "meanNs": 11589.1565, "allocatedBytes": 6824 }, "Digits=200": { - "meanNs": 271426.4561, + "meanNs": 275856.5164, "allocatedBytes": 126248 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 606.5947, + "meanNs": 632.3342, "allocatedBytes": 160 }, "Digits=30": { - "meanNs": 5412.4328, + "meanNs": 5474.5022, "allocatedBytes": 3200 }, "Digits=200": { - "meanNs": 128128.2382, + "meanNs": 128303.9274, "allocatedBytes": 54752 } }, "ConversionBenchmarks.FromDouble": { "": { - "meanNs": 1680.0394, + "meanNs": 1691.4001, "allocatedBytes": 1161 } }, "SignificanceBenchmarks.ReduceToThree": { "Digits=8": { - "meanNs": 301.3614, + "meanNs": 296.9042, "allocatedBytes": 80 }, "Digits=30": { - "meanNs": 1668.0534, + "meanNs": 1778.1202, "allocatedBytes": 1440 }, "Digits=200": { - "meanNs": 17373.0794, + "meanNs": 19180.7647, "allocatedBytes": 26944 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 518.674, + "meanNs": 521.9318, "allocatedBytes": 80 }, "Digits=30": { - "meanNs": 4660.0772, + "meanNs": 4757.4624, "allocatedBytes": 3040 }, "Digits=200": { - "meanNs": 85208.2083, + "meanNs": 88375.1774, "allocatedBytes": 54384 } } @@ -282,96 +354,120 @@ "date": "2026-09-14", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 452.8087, + "baselineNs": 456.0134, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.988, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3546, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.SignificantAdd": { + "": { + "meanNs": 1336.198, + "allocatedBytes": 512 + } + }, + "AbstractionCostBenchmarks.SignificantMultiply": { + "": { + "meanNs": 8052.555, + "allocatedBytes": 3577 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 629.4364, + "meanNs": 633.5077, "allocatedBytes": 160 }, "Digits=30": { - "meanNs": 5265.3493, + "meanNs": 5567.7348, "allocatedBytes": 3160 }, "Digits=200": { - "meanNs": 129051.806, + "meanNs": 130127.1917, "allocatedBytes": 54688 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 4367.9198, + "meanNs": 4388.8266, "allocatedBytes": 2298 }, "Digits=30": { - "meanNs": 9094.7489, + "meanNs": 9228.9665, "allocatedBytes": 5500 }, "Digits=200": { - "meanNs": 132643.3647, + "meanNs": 136023.1815, "allocatedBytes": 57158 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 1414.4, + "meanNs": 1419.9041, "allocatedBytes": 424 }, "Digits=30": { - "meanNs": 11243.8029, + "meanNs": 11698.694, "allocatedBytes": 6824 }, "Digits=200": { - "meanNs": 270934.7518, + "meanNs": 277342.7196, "allocatedBytes": 126248 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 610.7298, + "meanNs": 612.1539, "allocatedBytes": 160 }, "Digits=30": { - "meanNs": 5261.6772, + "meanNs": 5313.1919, "allocatedBytes": 3200 }, "Digits=200": { - "meanNs": 128266.5607, + "meanNs": 131867.3671, "allocatedBytes": 54752 } }, "ConversionBenchmarks.FromDouble": { "": { - "meanNs": 1636.9918, + "meanNs": 1733.104, "allocatedBytes": 1161 } }, "SignificanceBenchmarks.ReduceToThree": { "Digits=8": { - "meanNs": 292.872, + "meanNs": 296.2449, "allocatedBytes": 80 }, "Digits=30": { - "meanNs": 1685.0539, + "meanNs": 1792.1889, "allocatedBytes": 1440 }, "Digits=200": { - "meanNs": 17588.527, + "meanNs": 18418.2284, "allocatedBytes": 26944 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 509.4118, + "meanNs": 513.1928, "allocatedBytes": 80 }, "Digits=30": { - "meanNs": 4703.075, + "meanNs": 4854.3996, "allocatedBytes": 3040 }, "Digits=200": { - "meanNs": 87108.5511, + "meanNs": 90018.4517, "allocatedBytes": 54384 } } @@ -383,96 +479,120 @@ "date": "2026-09-15", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 452.8087, + "baselineNs": 456.0134, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9837, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.364, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.SignificantAdd": { + "": { + "meanNs": 88.2572, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.SignificantMultiply": { + "": { + "meanNs": 750.9037, + "allocatedBytes": 277 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 35.6765, + "meanNs": 35.4167, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 118.0456, + "meanNs": 121.9089, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 336.1947, + "meanNs": 334.5969, "allocatedBytes": 112 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 613.5962, + "meanNs": 636.7281, "allocatedBytes": 248 }, "Digits=30": { - "meanNs": 938.4486, + "meanNs": 937.7798, "allocatedBytes": 312 }, "Digits=200": { - "meanNs": 3370.5546, + "meanNs": 3485.3008, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 155.5987, + "meanNs": 160.3195, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 755.4034, + "meanNs": 910.2595, "allocatedBytes": 248 }, "Digits=200": { - "meanNs": 2920.5572, + "meanNs": 3040.5299, "allocatedBytes": 528 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 9.2312, + "meanNs": 7.9721, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 11.4472, + "meanNs": 11.7984, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 11.9296, + "meanNs": 11.7882, "allocatedBytes": 0 } }, "ConversionBenchmarks.FromDouble": { "": { - "meanNs": 359.4532, + "meanNs": 361.3465, "allocatedBytes": 0 } }, "SignificanceBenchmarks.ReduceToThree": { "Digits=8": { - "meanNs": 61.8836, + "meanNs": 63.6515, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 157.9816, + "meanNs": 157.669, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 211.8753, + "meanNs": 223.7146, "allocatedBytes": 224 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 195.492, + "meanNs": 187.166, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 490.0903, + "meanNs": 484.0393, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 3057.2608, + "meanNs": 2499.3696, "allocatedBytes": 112 } } @@ -484,96 +604,120 @@ "date": "2026-09-16", "cpu": "Intel Xeon Processor 2.80GHz", "runtime": ".NET 10.0.12 (10.0.12, 10.0.1226.42308)", - "baselineNs": 452.8087, + "baselineNs": 456.0134, "runId": "local-seed", "benchmarks": { + "AbstractionCostBenchmarks.BareAdd": { + "": { + "meanNs": 0.9927, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.BareMultiply": { + "": { + "meanNs": 1.3598, + "allocatedBytes": 0 + } + }, + "AbstractionCostBenchmarks.SignificantAdd": { + "": { + "meanNs": 89.2425, + "allocatedBytes": 32 + } + }, + "AbstractionCostBenchmarks.SignificantMultiply": { + "": { + "meanNs": 774.1825, + "allocatedBytes": 277 + } + }, "ArithmeticBenchmarks.Add": { "Digits=8": { - "meanNs": 35.6456, + "meanNs": 35.5057, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 123.9326, + "meanNs": 121.1506, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 345.8313, + "meanNs": 333.1362, "allocatedBytes": 112 } }, "ArithmeticBenchmarks.Divide": { "Digits=8": { - "meanNs": 623.6429, + "meanNs": 627.9404, "allocatedBytes": 248 }, "Digits=30": { - "meanNs": 943.3598, + "meanNs": 941.3271, "allocatedBytes": 312 }, "Digits=200": { - "meanNs": 3441.8766, + "meanNs": 3400.9066, "allocatedBytes": 528 } }, "ArithmeticBenchmarks.Multiply": { "Digits=8": { - "meanNs": 159.6749, + "meanNs": 159.2728, "allocatedBytes": 32 }, "Digits=30": { - "meanNs": 769.6604, + "meanNs": 782.0995, "allocatedBytes": 248 }, "Digits=200": { - "meanNs": 2986.04, + "meanNs": 2954.3011, "allocatedBytes": 528 } }, "ComparisonBenchmarks.CompareTo": { "Digits=8": { - "meanNs": 8.7894, + "meanNs": 8.77, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 11.7078, + "meanNs": 11.8088, "allocatedBytes": 0 }, "Digits=200": { - "meanNs": 12.4922, + "meanNs": 11.5867, "allocatedBytes": 0 } }, "ConversionBenchmarks.FromDouble": { "": { - "meanNs": 369.7793, + "meanNs": 364.5263, "allocatedBytes": 0 } }, "SignificanceBenchmarks.ReduceToThree": { "Digits=8": { - "meanNs": 62.995, + "meanNs": 62.5504, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 164.9354, + "meanNs": 166.2186, "allocatedBytes": 80 }, "Digits=200": { - "meanNs": 204.6833, + "meanNs": 220.0857, "allocatedBytes": 224 } }, "TextBenchmarks.Parse": { "Digits=8": { - "meanNs": 186.0136, + "meanNs": 183.7853, "allocatedBytes": 0 }, "Digits=30": { - "meanNs": 485.5936, + "meanNs": 476.9275, "allocatedBytes": 40 }, "Digits=200": { - "meanNs": 2372.9787, + "meanNs": 2462.7673, "allocatedBytes": 112 } } diff --git a/docs/benchmarks/performance-dark.svg b/docs/benchmarks/performance-dark.svg index 453eb63..94e6499 100644 --- a/docs/benchmarks/performance-dark.svg +++ b/docs/benchmarks/performance-dark.svg @@ -1,4 +1,4 @@ - + - + SignificantNumber performance by release 6 releases · newest 2.0.1 · 2026-09-16 @@ -95,79 +95,104 @@ Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster. Add (30 digits) - - - - - - + + + + + + -0.274× -12.2× +0.266× +11.9× Multiply (30 digits) - + - - - - - -1.70× -24.9× + + + + + +1.72× +25.8× Divide (30 digits) - - - - - - - -2.08× -19.5× + + + + + + + +2.06× +20.6× CompareTo (30 digits) - - - - - + + + + + 0.0259× -11.0× +12.2× Reduce to 3 (30 digits) - - + + - - -0.364× -3.68× + + +0.365× +3.90× Parse (30 digits) - - - - - + + + + + - -1.07× -10.8× + +1.05× +10.4× From double - + - - - - - -0.817× -3.77× -releases, oldest to newest: 1.3.0 → 1.4.0 → 1.4.20 → 1.4.40 → 2.0.0 → 2.0.1 -Measured on Intel Xeon Processor 2.80GHz. Full tables: SignificantNumber.Benchmarks. + + + + + +0.799× +3.82× + +Cost over the same arithmetic on a bare double +Divided by the identical loop on a double, measured beside it. This is the price of the precision, so it is well above 1 and belongs there; what matters is that it stays put. +Add + + + + + + + + +89.9× +1378× +Multiply + + + + + + + + +569× +5810× +releases, oldest to newest: 1.3.0 → 1.4.0 → 1.4.20 → 1.4.40 → 2.0.0 → 2.0.1 +Measured on Intel Xeon Processor 2.80GHz. Full tables: SignificantNumber.Benchmarks. diff --git a/docs/benchmarks/performance.svg b/docs/benchmarks/performance.svg index 660e489..f813059 100644 --- a/docs/benchmarks/performance.svg +++ b/docs/benchmarks/performance.svg @@ -1,4 +1,4 @@ - + - + SignificantNumber performance by release 6 releases · newest 2.0.1 · 2026-09-16 @@ -95,79 +95,104 @@ Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster. Add (30 digits) - - - - - - + + + + + + -0.274× -12.2× +0.266× +11.9× Multiply (30 digits) - + - - - - - -1.70× -24.9× + + + + + +1.72× +25.8× Divide (30 digits) - - - - - - - -2.08× -19.5× + + + + + + + +2.06× +20.6× CompareTo (30 digits) - - - - - + + + + + 0.0259× -11.0× +12.2× Reduce to 3 (30 digits) - - + + - - -0.364× -3.68× + + +0.365× +3.90× Parse (30 digits) - - - - - + + + + + - -1.07× -10.8× + +1.05× +10.4× From double - + - - - - - -0.817× -3.77× -releases, oldest to newest: 1.3.0 → 1.4.0 → 1.4.20 → 1.4.40 → 2.0.0 → 2.0.1 -Measured on Intel Xeon Processor 2.80GHz. Full tables: SignificantNumber.Benchmarks. + + + + + +0.799× +3.82× + +Cost over the same arithmetic on a bare double +Divided by the identical loop on a double, measured beside it. This is the price of the precision, so it is well above 1 and belongs there; what matters is that it stays put. +Add + + + + + + + + +89.9× +1378× +Multiply + + + + + + + + +569× +5810× +releases, oldest to newest: 1.3.0 → 1.4.0 → 1.4.20 → 1.4.40 → 2.0.0 → 2.0.1 +Measured on Intel Xeon Processor 2.80GHz. Full tables: SignificantNumber.Benchmarks. diff --git a/scripts/benchmark-history.cs b/scripts/benchmark-history.cs index 34e88a1..d47d81a 100644 --- a/scripts/benchmark-history.cs +++ b/scripts/benchmark-history.cs @@ -49,12 +49,39 @@ private static readonly (string Key, string? Parameters, string Label)[] Headlin /// private static readonly Dictionary Themes = new(StringComparer.Ordinal) { - ["light"] = new("#fcfcfb", "#0b0b0b", "#52514e", "#e4e3df", "#2a78d6", "#eb6834"), - ["dark"] = new("#1a1a19", "#ffffff", "#c3c2b7", "#333330", "#3987e5", "#d95926"), + ["light"] = new("#fcfcfb", "#0b0b0b", "#52514e", "#e4e3df", "#2a78d6", "#eb6834", "#2e8b57"), + ["dark"] = new("#1a1a19", "#ffffff", "#c3c2b7", "#333330", "#3987e5", "#d95926", "#3faa71"), }; private sealed record Theme( - string Surface, string Ink, string Muted, string Grid, string Alloc, string Time); + string Surface, string Ink, string Muted, string Grid, string Alloc, string Time, string Cost); + + /// Which of the three things a section draws. + private enum Measure + { + /// Bytes allocated per operation, as stored. + Allocation, + + /// Time, divided by the reference workload from the same job. + Time, + + /// Time, divided by the paired bare-double benchmark from the same entry. + Cost, + } + + /// + /// The paired benchmarks the cost section draws, as (measured, baseline, label). + /// + /// + /// Both halves are stored like any other benchmark; the ratio is computed here rather than + /// recorded, so an entry gathered before this section existed still draws once its run + /// includes the pair, and no history has to be rewritten to change what the section shows. + /// + private static readonly (string Key, string Baseline, string Label)[] CostHeadline = + [ + ("AbstractionCostBenchmarks.SignificantAdd", "AbstractionCostBenchmarks.BareAdd", "Add"), + ("AbstractionCostBenchmarks.SignificantMultiply", "AbstractionCostBenchmarks.BareMultiply", "Multiply"), + ]; internal static int Run(string[] args) { @@ -402,16 +429,17 @@ private static string Draw(JsonArray entries, Theme theme) string[] labels = [.. entries.Select(entry => entry!["version"]?.GetValue() ?? "?")]; int width = Left + (Columns * CellWidth) + 24; int rows = (Headline.Length + Columns - 1) / Columns; - int height = 72 + (((34 + (rows * CellHeight)) * 2) + 54); + int costRows = (CostHeadline.Length + Columns - 1) / Columns; + int height = 72 + ((34 + (rows * CellHeight)) * 2) + 34 + (costRows * CellHeight) + 54; StringBuilder svg = new(); Preamble(svg, theme, width, height, entries); int y = 72; - foreach (bool isTime in (bool[])[false, true]) + foreach (Measure measure in (Measure[])[Measure.Allocation, Measure.Time, Measure.Cost]) { - Section(svg, theme, entries, labels.Length, y, isTime); - y += 34 + (rows * CellHeight); + Section(svg, theme, entries, labels.Length, y, measure); + y += 34 + ((measure == Measure.Cost ? costRows : rows) * CellHeight); } Footer(svg, entries, labels, y - 4); @@ -441,20 +469,53 @@ private static void Preamble(StringBuilder svg, Theme theme, int width, int heig svg.AppendLine(CultureInfo.InvariantCulture, $"""{entries.Count} releases · newest {Escape(latest["version"]?.GetValue() ?? "?")}{suffix}"""); } - private static void Section(StringBuilder svg, Theme theme, JsonArray entries, int points, int y, bool isTime) + private static void Section(StringBuilder svg, Theme theme, JsonArray entries, int points, int y, Measure measure) { - string colour = isTime ? theme.Time : theme.Alloc; - string title = isTime - ? "Time, as a multiple of a fixed reference workload" - : "Allocated bytes per operation"; - string note = isTime - ? "Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster." - : "Deterministic: the same code allocates the same bytes on any machine."; + string colour = measure switch + { + Measure.Time => theme.Time, + Measure.Cost => theme.Cost, + _ => theme.Alloc, + }; + string title = measure switch + { + Measure.Time => "Time, as a multiple of a fixed reference workload", + Measure.Cost => "Cost over the same arithmetic on a bare double", + _ => "Allocated bytes per operation", + }; + string note = measure switch + { + Measure.Time => "Divided by a reference loop measured in the same job, which cancels most of the difference between CI runners. Lower is faster.", + Measure.Cost => "Divided by the identical loop on a double, measured beside it. This is the price of the precision, so it is well above 1 and belongs there; what matters is that it stays put.", + _ => "Deterministic: the same code allocates the same bytes on any machine.", + }; svg.AppendLine(CultureInfo.InvariantCulture, $""""""); svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(title)}"""); svg.AppendLine(CultureInfo.InvariantCulture, $"""{Escape(note)}"""); + if (measure == Measure.Cost) + { + for (int position = 0; position < CostHeadline.Length; position++) + { + (string key, string baseline, string label) = CostHeadline[position]; + double?[] values = [.. entries.Select(entry => Cost(entry!, key, baseline))]; + Panel( + svg, + Left + (position % Columns * CellWidth), + y + 26 + (position / Columns * CellHeight), + label, + points, + values, + true, + colour, + theme); + } + + return; + } + + bool isTime = measure == Measure.Time; for (int position = 0; position < Headline.Length; position++) { (string key, string? parameters, string label) = Headline[position]; @@ -472,6 +533,25 @@ private static void Section(StringBuilder svg, Theme theme, JsonArray entries, i } } + /// + /// One benchmark's mean divided by the mean of the bare-double benchmark beside it. + /// + /// + /// Both were measured in the same job on the same machine, so unlike the time section this + /// needs no reference workload to be comparable across runs: the denominator is the reference. + /// + private static double? Cost(JsonNode entry, string key, string baseline) + { + double? measured = Mean(entry, key); + double? divisor = Mean(entry, baseline); + return measured is not null && divisor is > 0 ? measured / divisor : null; + } + + private static double? Mean(JsonNode entry, string key) => + entry["benchmarks"]?[key] is JsonObject cases && cases.Count > 0 + ? cases.First().Value?["meanNs"]?.GetValue() + : null; + private static double? Value(JsonNode entry, string key, string? parameters, bool isTime) { if (entry["benchmarks"]?[key] is not JsonObject cases || cases.Count == 0)