diff --git a/src/TriasDev.Tabular/Analysis/ColumnFacts.cs b/src/TriasDev.Tabular/Analysis/ColumnFacts.cs
index 3083ad8..fee82d9 100644
--- a/src/TriasDev.Tabular/Analysis/ColumnFacts.cs
+++ b/src/TriasDev.Tabular/Analysis/ColumnFacts.cs
@@ -140,4 +140,16 @@ public sealed record ColumnFacts
/// the distinct budget ran out, since a set cannot be complete when the count behind it is not.
///
public required bool DistinctValuesAreComplete { get; init; }
+
+ ///
+ /// Every value of with the number of rows holding it, in the same
+ /// order; empty whenever is false.
+ ///
+ ///
+ /// A tally, not a sample: the counts add up to . What lets the precheck
+ /// say how many rows a rule fails rather than how many values, and a screen say "3 rows hold a
+ /// material that is not allowed" (#63). Values are counted as written, trimmed, so Wood and
+ /// wood are two entries a case-insensitive caller adds up.
+ ///
+ public IReadOnlyList DistinctValueCounts { get; init => field = Equatable.List(value); } = Equatable.Empty();
}
diff --git a/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs b/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs
index a781ffa..684faa5 100644
--- a/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs
+++ b/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs
@@ -217,6 +217,9 @@ public ColumnFacts ToFacts(string? preferredDecimalSeparator = null)
Samples = [.. _firstValues],
DistinctValues = _distinctValuesComplete ? [.. _distinctValues] : [],
DistinctValuesAreComplete = _distinctValuesComplete,
+ DistinctValueCounts = _distinctValuesComplete
+ ? [.. _distinctValues.Select(v => new ValueFrequency { Value = v, Count = _frequencies.GetValueOrDefault(v) })]
+ : [],
};
}
@@ -372,6 +375,7 @@ internal int StopCounting()
_distinctValuesComplete = false;
_distinctValues.Clear();
_distinctValues.TrimExcess();
+ ShrinkFrequencies();
return held;
}
@@ -424,6 +428,33 @@ private void TrackDistinct(string text)
_distinctValuesComplete = false;
_distinctValues.Clear();
_distinctValues.TrimExcess();
+ ShrinkFrequencies();
+ }
+
+ ///
+ /// Takes the frequency tally back to a sample once the column's values stop being kept whole: its
+ /// most frequent keys, as many as the sample holds, so the lookup every value pays is against the
+ /// small dictionary again.
+ ///
+ private void ShrinkFrequencies()
+ {
+ if (_frequencies.Count <= _options.FrequencySampleSize)
+ {
+ return;
+ }
+
+ KeyValuePair[] kept = [.. _frequencies
+ .OrderByDescending(pair => pair.Value)
+ .ThenBy(pair => pair.Key, StringComparer.Ordinal)
+ .Take(_options.FrequencySampleSize)];
+
+ _frequencies.Clear();
+ _frequencies.TrimExcess();
+
+ foreach ((string key, int count) in kept)
+ {
+ _frequencies[key] = count;
+ }
}
private void TrackFrequency(string text)
@@ -434,7 +465,14 @@ private void TrackFrequency(string text)
return;
}
- if (_frequencies.Count < _options.FrequencySampleSize)
+ // While every distinct value is kept, every one is counted, which is what makes the counts a
+ // column's full tally rather than a sample (#63). The lookup above is paid on every value
+ // either way; only the keys cost, and they are the strings already kept.
+ int cap = _distinctValuesComplete
+ ? Math.Max(_options.FrequencySampleSize, _options.RetainedDistinctValues)
+ : _options.FrequencySampleSize;
+
+ if (_frequencies.Count < cap)
{
_frequencies[text] = 1;
}
diff --git a/src/TriasDev.Tabular/Import/MappingPrecheck.cs b/src/TriasDev.Tabular/Import/MappingPrecheck.cs
index 214bcb7..fe469b0 100644
--- a/src/TriasDev.Tabular/Import/MappingPrecheck.cs
+++ b/src/TriasDev.Tabular/Import/MappingPrecheck.cs
@@ -569,10 +569,15 @@ .. read.Where(r => r.Readable && !constraint.IsSatisfiedBy(r.Value)).Select(r =>
int readable = read.Count(r => r.Readable);
bool none = failing.Count == readable && CannotBeSatisfiedByAnyRow(field, binding, facts);
+ // The rows, not only the values: each failing value's tally, where the facts hold one (#63).
+ Dictionary tally = facts.DistinctValueCounts.ToDictionary(v => v.Value, v => v.Count, StringComparer.Ordinal);
+ int? rows = tally.Count > 0 ? failing.Sum(v => tally.GetValueOrDefault(v)) : null;
+
add(
constraint.Code,
none ? PrecheckSeverity.Blocking : PrecheckSeverity.Warning,
new Evidence(
+ rows,
Examples: [.. failing.Order(StringComparer.Ordinal).Take(5)],
Arguments: Args(
(PrecheckArguments.Reason, none ? PrecheckReasons.NoRowCanSatisfy : PrecheckReasons.ValuesFail),
diff --git a/src/TriasDev.Tabular/PublicAPI.Unshipped.txt b/src/TriasDev.Tabular/PublicAPI.Unshipped.txt
index ec7da17..e3ffd2f 100644
--- a/src/TriasDev.Tabular/PublicAPI.Unshipped.txt
+++ b/src/TriasDev.Tabular/PublicAPI.Unshipped.txt
@@ -5,3 +5,5 @@ TriasDev.Tabular.CultureParseCounts.DatesWithOwnSeparator.get -> int
TriasDev.Tabular.CultureParseCounts.DatesWithOwnSeparator.init -> void
TriasDev.Tabular.ColumnFacts.NumberReadingsDisagree.get -> bool
TriasDev.Tabular.ColumnFacts.NumberReadingsDisagree.init -> void
+TriasDev.Tabular.ColumnFacts.DistinctValueCounts.get -> System.Collections.Generic.IReadOnlyList!
+TriasDev.Tabular.ColumnFacts.DistinctValueCounts.init -> void
diff --git a/tests/TriasDev.Tabular.Tests/Analysis/DistinctValueCountsTests.cs b/tests/TriasDev.Tabular.Tests/Analysis/DistinctValueCountsTests.cs
new file mode 100644
index 0000000..e1da278
--- /dev/null
+++ b/tests/TriasDev.Tabular.Tests/Analysis/DistinctValueCountsTests.cs
@@ -0,0 +1,89 @@
+using System.Globalization;
+using System.Text;
+
+using TriasDev.Tabular.Csv;
+
+using Xunit;
+
+namespace TriasDev.Tabular.Tests.Analysis;
+
+///
+/// A column whose distinct values are all kept also says how many rows hold each, and the precheck
+/// turns that into the exact number of rows a rule fails (#63).
+///
+public sealed class DistinctValueCountsTests
+{
+ private static CancellationToken Token => TestContext.Current.CancellationToken;
+
+ private static FileProfile Profile(string csv, AnalysisOptions? options = null)
+ {
+ using CsvCursor cursor = new(new MemoryStream(Encoding.UTF8.GetBytes(csv)), "t.csv");
+ return TabularAnalyzer.Analyze(cursor, options, cancellationToken: Token);
+ }
+
+ private const string Buildings = "id;material;storeys\n1;Wood;2\n2;wood;3\n3;Steel;40\n4;Concrete;5\n5;Straw;2\n6;;1\n7;Wood;-1\n";
+
+ [Fact]
+ public void CountsTheRowsOfEveryDistinctValueInTheOrderFirstSeen()
+ {
+ ColumnFacts material = Profile(Buildings).Sheets[0].Columns[1].Facts;
+
+ Assert.True(material.DistinctValuesAreComplete);
+ Assert.Equal(
+ [("Wood", 2), ("wood", 1), ("Steel", 1), ("Concrete", 1), ("Straw", 1)],
+ material.DistinctValueCounts.Select(v => (v.Value, v.Count)));
+ Assert.Equal(material.NonEmptyCount, material.DistinctValueCounts.Sum(v => v.Count));
+ }
+
+ [Fact]
+ public void CountsBeyondTheFrequencySampleWhileTheSetIsComplete()
+ {
+ // 300 distinct values, each twice: past the 64-key sample, within the 1,000 kept.
+ StringBuilder csv = new("code\n");
+
+ for (int i = 0; i < 600; i++)
+ {
+ csv.Append("c").Append((i % 300).ToString(CultureInfo.InvariantCulture)).Append('\n');
+ }
+
+ ColumnFacts code = Profile(csv.ToString()).Sheets[0].Columns[0].Facts;
+
+ Assert.Equal(300, code.DistinctValueCounts.Count);
+ Assert.All(code.DistinctValueCounts, v => Assert.Equal(2, v.Count));
+ }
+
+ [Fact]
+ public void GivesNoCountsOnceTheSetIsIncomplete()
+ {
+ ColumnFacts material = Profile(Buildings, new AnalysisOptions { RetainedDistinctValues = 3 }).Sheets[0].Columns[1].Facts;
+
+ Assert.False(material.DistinctValuesAreComplete);
+ Assert.Empty(material.DistinctValueCounts);
+ }
+
+ [Fact]
+ public void ReportsTheExactRowsARuleFails()
+ {
+ TextImportField material = ImportField.Text("material").AllowedValues(["wood", "steel", "concrete"], ignoreCase: true);
+ IntegerImportField storeys = ImportField.Integer("storeys").AtLeast(0).AtMost(30);
+ MappingPlan plan = new()
+ {
+ Bindings =
+ [
+ new ColumnBinding { ColumnIndex = 1, FieldName = "material", Header = "material" },
+ new ColumnBinding { ColumnIndex = 2, FieldName = "storeys", Header = "storeys" },
+ ],
+ };
+
+ PrecheckResult result = MappingPrecheck.Check(plan, new ImportSchema { Fields = [material, storeys] }, Profile(Buildings));
+
+ PrecheckFinding notAllowed = Assert.Single(result.Findings, f => f.Code == ErrorCodes.Value.NotAllowed);
+ Assert.Equal(1, notAllowed.AffectedRows);
+ Assert.Equal(RowCountBound.Exact, notAllowed.AffectedRowsBound);
+
+ // One row below the floor, one above the ceiling: a finding for each rule.
+ PrecheckFinding[] outOfRange = [.. result.Findings.Where(f => f.Code == ErrorCodes.Value.OutOfRange)];
+ Assert.Equal([1, 1], outOfRange.Select(f => f.AffectedRows));
+ Assert.All(outOfRange, f => Assert.Equal(RowCountBound.Exact, f.AffectedRowsBound));
+ }
+}