feat: DistinctValueCounts, and exact AffectedRows for precheck rule findings

src/TriasDev.Tabular/Analysis/ColumnFacts.cs

@@ -140,4 +140,16 @@ public sealed record ColumnFacts
     /// the distinct budget ran out, since a set cannot be complete when the count behind it is not.
     /// </remarks>
     public required bool DistinctValuesAreComplete { get; init; }
+
+    /// <summary>
+    /// Every value of <see cref="DistinctValues"/> with the number of rows holding it, in the same
+    /// order; empty whenever <see cref="DistinctValuesAreComplete"/> is false.
+    /// </summary>
+    /// <remarks>
+    /// A tally, not a sample: the counts add up to <see cref="NonEmptyCount"/>. What lets the precheck
+    /// say how many rows a rule fails rather than how many values, and a screen say "3 rows hold a
+    /// material that is not allowed" (#63). Values are counted as written, trimmed, so <c>Wood</c> and
+    /// <c>wood</c> are two entries a case-insensitive caller adds up.
+    /// </remarks>
+    public IReadOnlyList<ValueFrequency> DistinctValueCounts { get; init => field = Equatable.List(value); } = Equatable.Empty<ValueFrequency>();
 }

src/TriasDev.Tabular/Analysis/ColumnProfiler.cs

@@ -217,6 +217,9 @@ public ColumnFacts ToFacts(string? preferredDecimalSeparator = null)
             Samples = [.. _firstValues],
             DistinctValues = _distinctValuesComplete ? [.. _distinctValues] : [],
             DistinctValuesAreComplete = _distinctValuesComplete,
+            DistinctValueCounts = _distinctValuesComplete
+                ? [.. _distinctValues.Select(v => new ValueFrequency { Value = v, Count = _frequencies.GetValueOrDefault(v) })]
+                : [],
         };
     }
 
@@ -372,6 +375,7 @@ internal int StopCounting()
         _distinctValuesComplete = false;
         _distinctValues.Clear();
         _distinctValues.TrimExcess();
+        ShrinkFrequencies();
         return held;
     }
 
@@ -424,6 +428,33 @@ private void TrackDistinct(string text)
         _distinctValuesComplete = false;
         _distinctValues.Clear();
         _distinctValues.TrimExcess();
+        ShrinkFrequencies();
+    }
+
+    /// <summary>
+    /// Takes the frequency tally back to a sample once the column's values stop being kept whole: its
+    /// most frequent keys, as many as the sample holds, so the lookup every value pays is against the
+    /// small dictionary again.
+    /// </summary>
+    private void ShrinkFrequencies()
+    {
+        if (_frequencies.Count <= _options.FrequencySampleSize)
+        {
+            return;
+        }
+
+        KeyValuePair<string, int>[] kept = [.. _frequencies
+            .OrderByDescending(pair => pair.Value)
+            .ThenBy(pair => pair.Key, StringComparer.Ordinal)
+            .Take(_options.FrequencySampleSize)];
+
+        _frequencies.Clear();
+        _frequencies.TrimExcess();
+
+        foreach ((string key, int count) in kept)
+        {
+            _frequencies[key] = count;
+        }
     }
 
     private void TrackFrequency(string text)
@@ -434,7 +465,14 @@ private void TrackFrequency(string text)
             return;
         }
 
-        if (_frequencies.Count < _options.FrequencySampleSize)
+        // While every distinct value is kept, every one is counted, which is what makes the counts a
+        // column's full tally rather than a sample (#63). The lookup above is paid on every value
+        // either way; only the keys cost, and they are the strings already kept.
+        int cap = _distinctValuesComplete
+            ? Math.Max(_options.FrequencySampleSize, _options.RetainedDistinctValues)
+            : _options.FrequencySampleSize;
+
+        if (_frequencies.Count < cap)
         {
             _frequencies[text] = 1;
         }

src/TriasDev.Tabular/Import/MappingPrecheck.cs

@@ -569,10 +569,15 @@ .. read.Where(r => r.Readable && !constraint.IsSatisfiedBy(r.Value)).Select(r =>
         int readable = read.Count(r => r.Readable);
         bool none = failing.Count == readable && CannotBeSatisfiedByAnyRow(field, binding, facts);
 
+        // The rows, not only the values: each failing value's tally, where the facts hold one (#63).
+        Dictionary<string, int> tally = facts.DistinctValueCounts.ToDictionary(v => v.Value, v => v.Count, StringComparer.Ordinal);
+        int? rows = tally.Count > 0 ? failing.Sum(v => tally.GetValueOrDefault(v)) : null;
+
         add(
             constraint.Code,
             none ? PrecheckSeverity.Blocking : PrecheckSeverity.Warning,
             new Evidence(
+                rows,
                 Examples: [.. failing.Order(StringComparer.Ordinal).Take(5)],
                 Arguments: Args(
                     (PrecheckArguments.Reason, none ? PrecheckReasons.NoRowCanSatisfy : PrecheckReasons.ValuesFail),

src/TriasDev.Tabular/PublicAPI.Unshipped.txt

@@ -5,3 +5,5 @@ TriasDev.Tabular.CultureParseCounts.DatesWithOwnSeparator.get -> int
 TriasDev.Tabular.CultureParseCounts.DatesWithOwnSeparator.init -> void
 TriasDev.Tabular.ColumnFacts.NumberReadingsDisagree.get -> bool
 TriasDev.Tabular.ColumnFacts.NumberReadingsDisagree.init -> void
+TriasDev.Tabular.ColumnFacts.DistinctValueCounts.get -> System.Collections.Generic.IReadOnlyList<TriasDev.Tabular.ValueFrequency>!
+TriasDev.Tabular.ColumnFacts.DistinctValueCounts.init -> void

tests/TriasDev.Tabular.Tests/Analysis/DistinctValueCountsTests.cs

@@ -0,0 +1,89 @@
+using System.Globalization;
+using System.Text;
+
+using TriasDev.Tabular.Csv;
+
+using Xunit;
+
+namespace TriasDev.Tabular.Tests.Analysis;
+
+/// <summary>
+/// A column whose distinct values are all kept also says how many rows hold each, and the precheck
+/// turns that into the exact number of rows a rule fails (#63).
+/// </summary>
+public sealed class DistinctValueCountsTests
+{
+    private static CancellationToken Token => TestContext.Current.CancellationToken;
+
+    private static FileProfile Profile(string csv, AnalysisOptions? options = null)
+    {
+        using CsvCursor cursor = new(new MemoryStream(Encoding.UTF8.GetBytes(csv)), "t.csv");
+        return TabularAnalyzer.Analyze(cursor, options, cancellationToken: Token);
+    }
+
+    private const string Buildings = "id;material;storeys\n1;Wood;2\n2;wood;3\n3;Steel;40\n4;Concrete;5\n5;Straw;2\n6;;1\n7;Wood;-1\n";
+
+    [Fact]
+    public void CountsTheRowsOfEveryDistinctValueInTheOrderFirstSeen()
+    {
+        ColumnFacts material = Profile(Buildings).Sheets[0].Columns[1].Facts;
+
+        Assert.True(material.DistinctValuesAreComplete);
+        Assert.Equal(
+            [("Wood", 2), ("wood", 1), ("Steel", 1), ("Concrete", 1), ("Straw", 1)],
+            material.DistinctValueCounts.Select(v => (v.Value, v.Count)));
+        Assert.Equal(material.NonEmptyCount, material.DistinctValueCounts.Sum(v => v.Count));
+    }
+
+    [Fact]
+    public void CountsBeyondTheFrequencySampleWhileTheSetIsComplete()
+    {
+        // 300 distinct values, each twice: past the 64-key sample, within the 1,000 kept.
+        StringBuilder csv = new("code\n");
+
+        for (int i = 0; i < 600; i++)
+        {
+            csv.Append("c").Append((i % 300).ToString(CultureInfo.InvariantCulture)).Append('\n');
+        }
+
+        ColumnFacts code = Profile(csv.ToString()).Sheets[0].Columns[0].Facts;
+
+        Assert.Equal(300, code.DistinctValueCounts.Count);
+        Assert.All(code.DistinctValueCounts, v => Assert.Equal(2, v.Count));
+    }
+
+    [Fact]
+    public void GivesNoCountsOnceTheSetIsIncomplete()
+    {
+        ColumnFacts material = Profile(Buildings, new AnalysisOptions { RetainedDistinctValues = 3 }).Sheets[0].Columns[1].Facts;
+
+        Assert.False(material.DistinctValuesAreComplete);
+        Assert.Empty(material.DistinctValueCounts);
+    }
+
+    [Fact]
+    public void ReportsTheExactRowsARuleFails()
+    {
+        TextImportField material = ImportField.Text("material").AllowedValues(["wood", "steel", "concrete"], ignoreCase: true);
+        IntegerImportField storeys = ImportField.Integer("storeys").AtLeast(0).AtMost(30);
+        MappingPlan plan = new()
+        {
+            Bindings =
+            [
+                new ColumnBinding { ColumnIndex = 1, FieldName = "material", Header = "material" },
+                new ColumnBinding { ColumnIndex = 2, FieldName = "storeys", Header = "storeys" },
+            ],
+        };
+
+        PrecheckResult result = MappingPrecheck.Check(plan, new ImportSchema { Fields = [material, storeys] }, Profile(Buildings));
+
+        PrecheckFinding notAllowed = Assert.Single(result.Findings, f => f.Code == ErrorCodes.Value.NotAllowed);
+        Assert.Equal(1, notAllowed.AffectedRows);
+        Assert.Equal(RowCountBound.Exact, notAllowed.AffectedRowsBound);
+
+        // One row below the floor, one above the ceiling: a finding for each rule.
+        PrecheckFinding[] outOfRange = [.. result.Findings.Where(f => f.Code == ErrorCodes.Value.OutOfRange)];
+        Assert.Equal([1, 1], outOfRange.Select(f => f.AffectedRows));
+        Assert.All(outOfRange, f => Assert.Equal(RowCountBound.Exact, f.AffectedRowsBound));
+    }
+}