From 6bc47a88ee94136248f0d7154431a2d0f36eff00 Mon Sep 17 00:00:00 2001 From: Slava Date: Tue, 29 Sep 2026 18:23:37 +0200 Subject: [PATCH] feat: count the rows of every distinct value, and give precheck rule findings their exact affected rows --- src/TriasDev.Tabular/Analysis/ColumnFacts.cs | 12 +++ .../Analysis/ColumnProfiler.cs | 40 ++++++++- .../Import/MappingPrecheck.cs | 5 ++ src/TriasDev.Tabular/PublicAPI.Unshipped.txt | 2 + .../Analysis/DistinctValueCountsTests.cs | 89 +++++++++++++++++++ 5 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 tests/TriasDev.Tabular.Tests/Analysis/DistinctValueCountsTests.cs diff --git a/src/TriasDev.Tabular/Analysis/ColumnFacts.cs b/src/TriasDev.Tabular/Analysis/ColumnFacts.cs index 3083ad8..fee82d9 100644 --- a/src/TriasDev.Tabular/Analysis/ColumnFacts.cs +++ b/src/TriasDev.Tabular/Analysis/ColumnFacts.cs @@ -140,4 +140,16 @@ public sealed record ColumnFacts /// the distinct budget ran out, since a set cannot be complete when the count behind it is not. /// public required bool DistinctValuesAreComplete { get; init; } + + /// + /// Every value of with the number of rows holding it, in the same + /// order; empty whenever is false. + /// + /// + /// A tally, not a sample: the counts add up to . What lets the precheck + /// say how many rows a rule fails rather than how many values, and a screen say "3 rows hold a + /// material that is not allowed" (#63). Values are counted as written, trimmed, so Wood and + /// wood are two entries a case-insensitive caller adds up. + /// + public IReadOnlyList DistinctValueCounts { get; init => field = Equatable.List(value); } = Equatable.Empty(); } diff --git a/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs b/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs index a781ffa..684faa5 100644 --- a/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs +++ b/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs @@ -217,6 +217,9 @@ public ColumnFacts ToFacts(string? preferredDecimalSeparator = null) Samples = [.. _firstValues], DistinctValues = _distinctValuesComplete ? [.. _distinctValues] : [], DistinctValuesAreComplete = _distinctValuesComplete, + DistinctValueCounts = _distinctValuesComplete + ? [.. _distinctValues.Select(v => new ValueFrequency { Value = v, Count = _frequencies.GetValueOrDefault(v) })] + : [], }; } @@ -372,6 +375,7 @@ internal int StopCounting() _distinctValuesComplete = false; _distinctValues.Clear(); _distinctValues.TrimExcess(); + ShrinkFrequencies(); return held; } @@ -424,6 +428,33 @@ private void TrackDistinct(string text) _distinctValuesComplete = false; _distinctValues.Clear(); _distinctValues.TrimExcess(); + ShrinkFrequencies(); + } + + /// + /// Takes the frequency tally back to a sample once the column's values stop being kept whole: its + /// most frequent keys, as many as the sample holds, so the lookup every value pays is against the + /// small dictionary again. + /// + private void ShrinkFrequencies() + { + if (_frequencies.Count <= _options.FrequencySampleSize) + { + return; + } + + KeyValuePair[] kept = [.. _frequencies + .OrderByDescending(pair => pair.Value) + .ThenBy(pair => pair.Key, StringComparer.Ordinal) + .Take(_options.FrequencySampleSize)]; + + _frequencies.Clear(); + _frequencies.TrimExcess(); + + foreach ((string key, int count) in kept) + { + _frequencies[key] = count; + } } private void TrackFrequency(string text) @@ -434,7 +465,14 @@ private void TrackFrequency(string text) return; } - if (_frequencies.Count < _options.FrequencySampleSize) + // While every distinct value is kept, every one is counted, which is what makes the counts a + // column's full tally rather than a sample (#63). The lookup above is paid on every value + // either way; only the keys cost, and they are the strings already kept. + int cap = _distinctValuesComplete + ? Math.Max(_options.FrequencySampleSize, _options.RetainedDistinctValues) + : _options.FrequencySampleSize; + + if (_frequencies.Count < cap) { _frequencies[text] = 1; } diff --git a/src/TriasDev.Tabular/Import/MappingPrecheck.cs b/src/TriasDev.Tabular/Import/MappingPrecheck.cs index 214bcb7..fe469b0 100644 --- a/src/TriasDev.Tabular/Import/MappingPrecheck.cs +++ b/src/TriasDev.Tabular/Import/MappingPrecheck.cs @@ -569,10 +569,15 @@ .. read.Where(r => r.Readable && !constraint.IsSatisfiedBy(r.Value)).Select(r => int readable = read.Count(r => r.Readable); bool none = failing.Count == readable && CannotBeSatisfiedByAnyRow(field, binding, facts); + // The rows, not only the values: each failing value's tally, where the facts hold one (#63). + Dictionary tally = facts.DistinctValueCounts.ToDictionary(v => v.Value, v => v.Count, StringComparer.Ordinal); + int? rows = tally.Count > 0 ? failing.Sum(v => tally.GetValueOrDefault(v)) : null; + add( constraint.Code, none ? PrecheckSeverity.Blocking : PrecheckSeverity.Warning, new Evidence( + rows, Examples: [.. failing.Order(StringComparer.Ordinal).Take(5)], Arguments: Args( (PrecheckArguments.Reason, none ? PrecheckReasons.NoRowCanSatisfy : PrecheckReasons.ValuesFail), diff --git a/src/TriasDev.Tabular/PublicAPI.Unshipped.txt b/src/TriasDev.Tabular/PublicAPI.Unshipped.txt index ec7da17..e3ffd2f 100644 --- a/src/TriasDev.Tabular/PublicAPI.Unshipped.txt +++ b/src/TriasDev.Tabular/PublicAPI.Unshipped.txt @@ -5,3 +5,5 @@ TriasDev.Tabular.CultureParseCounts.DatesWithOwnSeparator.get -> int TriasDev.Tabular.CultureParseCounts.DatesWithOwnSeparator.init -> void TriasDev.Tabular.ColumnFacts.NumberReadingsDisagree.get -> bool TriasDev.Tabular.ColumnFacts.NumberReadingsDisagree.init -> void +TriasDev.Tabular.ColumnFacts.DistinctValueCounts.get -> System.Collections.Generic.IReadOnlyList! +TriasDev.Tabular.ColumnFacts.DistinctValueCounts.init -> void diff --git a/tests/TriasDev.Tabular.Tests/Analysis/DistinctValueCountsTests.cs b/tests/TriasDev.Tabular.Tests/Analysis/DistinctValueCountsTests.cs new file mode 100644 index 0000000..e1da278 --- /dev/null +++ b/tests/TriasDev.Tabular.Tests/Analysis/DistinctValueCountsTests.cs @@ -0,0 +1,89 @@ +using System.Globalization; +using System.Text; + +using TriasDev.Tabular.Csv; + +using Xunit; + +namespace TriasDev.Tabular.Tests.Analysis; + +/// +/// A column whose distinct values are all kept also says how many rows hold each, and the precheck +/// turns that into the exact number of rows a rule fails (#63). +/// +public sealed class DistinctValueCountsTests +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + + private static FileProfile Profile(string csv, AnalysisOptions? options = null) + { + using CsvCursor cursor = new(new MemoryStream(Encoding.UTF8.GetBytes(csv)), "t.csv"); + return TabularAnalyzer.Analyze(cursor, options, cancellationToken: Token); + } + + private const string Buildings = "id;material;storeys\n1;Wood;2\n2;wood;3\n3;Steel;40\n4;Concrete;5\n5;Straw;2\n6;;1\n7;Wood;-1\n"; + + [Fact] + public void CountsTheRowsOfEveryDistinctValueInTheOrderFirstSeen() + { + ColumnFacts material = Profile(Buildings).Sheets[0].Columns[1].Facts; + + Assert.True(material.DistinctValuesAreComplete); + Assert.Equal( + [("Wood", 2), ("wood", 1), ("Steel", 1), ("Concrete", 1), ("Straw", 1)], + material.DistinctValueCounts.Select(v => (v.Value, v.Count))); + Assert.Equal(material.NonEmptyCount, material.DistinctValueCounts.Sum(v => v.Count)); + } + + [Fact] + public void CountsBeyondTheFrequencySampleWhileTheSetIsComplete() + { + // 300 distinct values, each twice: past the 64-key sample, within the 1,000 kept. + StringBuilder csv = new("code\n"); + + for (int i = 0; i < 600; i++) + { + csv.Append("c").Append((i % 300).ToString(CultureInfo.InvariantCulture)).Append('\n'); + } + + ColumnFacts code = Profile(csv.ToString()).Sheets[0].Columns[0].Facts; + + Assert.Equal(300, code.DistinctValueCounts.Count); + Assert.All(code.DistinctValueCounts, v => Assert.Equal(2, v.Count)); + } + + [Fact] + public void GivesNoCountsOnceTheSetIsIncomplete() + { + ColumnFacts material = Profile(Buildings, new AnalysisOptions { RetainedDistinctValues = 3 }).Sheets[0].Columns[1].Facts; + + Assert.False(material.DistinctValuesAreComplete); + Assert.Empty(material.DistinctValueCounts); + } + + [Fact] + public void ReportsTheExactRowsARuleFails() + { + TextImportField material = ImportField.Text("material").AllowedValues(["wood", "steel", "concrete"], ignoreCase: true); + IntegerImportField storeys = ImportField.Integer("storeys").AtLeast(0).AtMost(30); + MappingPlan plan = new() + { + Bindings = + [ + new ColumnBinding { ColumnIndex = 1, FieldName = "material", Header = "material" }, + new ColumnBinding { ColumnIndex = 2, FieldName = "storeys", Header = "storeys" }, + ], + }; + + PrecheckResult result = MappingPrecheck.Check(plan, new ImportSchema { Fields = [material, storeys] }, Profile(Buildings)); + + PrecheckFinding notAllowed = Assert.Single(result.Findings, f => f.Code == ErrorCodes.Value.NotAllowed); + Assert.Equal(1, notAllowed.AffectedRows); + Assert.Equal(RowCountBound.Exact, notAllowed.AffectedRowsBound); + + // One row below the floor, one above the ceiling: a finding for each rule. + PrecheckFinding[] outOfRange = [.. result.Findings.Where(f => f.Code == ErrorCodes.Value.OutOfRange)]; + Assert.Equal([1, 1], outOfRange.Select(f => f.AffectedRows)); + Assert.All(outOfRange, f => Assert.Equal(RowCountBound.Exact, f.AffectedRowsBound)); + } +}