fix(analysis): rank the date culture whose separator the values use, and flag disagreeing readings

src/TriasDev.Tabular/Analysis/ColumnFacts.cs

@@ -83,6 +83,18 @@ public sealed record ColumnFacts
     /// </remarks>
     public required bool? IsUnique { get; init; }
 
+    /// <summary>
+    /// Whether the cultures that read the most values as dates read some row as different dates.
+    /// </summary>
+    /// <remarks>
+    /// True for a column whose every day is twelve or less, written <c>11.01.2018</c> or
+    /// <c>11/01/2018</c>: German reads it as 11 January, American as 1 November, and every value fits
+    /// both. The hypotheses rank the culture whose separator the values use first, and the dates in
+    /// these facts follow it, but that is a preference, not evidence — this says so, so a screen can
+    /// ask rather than guess (#58).
+    /// </remarks>
+    public bool DateReadingsDisagree { get; init; }
+
     /// <summary>
     /// A bounded sample of values with how often each was seen.
     /// </summary>

src/TriasDev.Tabular/Analysis/ColumnProfiler.cs

@@ -208,6 +208,7 @@ public ColumnFacts ToFacts()
             DistinctCount = _distinctIsExact ? _distinctHashes.Count : _distinctCountAtStop,
             DistinctCountIsExact = _distinctIsExact,
             IsUnique = Unique(),
+            DateReadingsDisagree = DateReadingsDisagree(),
             DistinctSamples = TopFrequencies(),
             Samples = [.. _firstValues],
             DistinctValues = _distinctValuesComplete ? [.. _distinctValues] : [],
@@ -228,8 +229,12 @@ private CultureAccumulator BestCulture()
 
         foreach (CultureAccumulator culture in _cultures)
         {
+            // The third criterion is #58: of cultures reading as many dates, the one whose separator the
+            // dates are written with, so 11.01.2018 publishes January rather than November.
             if (culture.NumericCount > best.NumericCount
-                || (culture.NumericCount == best.NumericCount && culture.DateCount > best.DateCount))
+                || (culture.NumericCount == best.NumericCount && culture.DateCount > best.DateCount)
+                || (culture.NumericCount == best.NumericCount && culture.DateCount == best.DateCount
+                    && culture.DatesWithOwnSeparator > best.DatesWithOwnSeparator))
             {
                 best = culture;
             }
@@ -238,6 +243,40 @@ private CultureAccumulator BestCulture()
         return best;
     }
 
+    /// <summary>
+    /// Whether two cultures that read the most dates read some row as different dates.
+    /// </summary>
+    private bool DateReadingsDisagree()
+    {
+        int most = _cultures.Max(c => c.DateCount);
+
+        if (most == 0)
+        {
+            return false;
+        }
+
+        ulong? first = null;
+
+        foreach (CultureAccumulator culture in _cultures)
+        {
+            if (culture.DateCount != most)
+            {
+                continue;
+            }
+
+            if (first is null)
+            {
+                first = culture.DateFingerprint;
+            }
+            else if (culture.DateFingerprint != first)
+            {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     /// <summary>
     /// Whether this column can identify its rows.
     /// </summary>
@@ -390,15 +429,40 @@ private sealed class CultureAccumulator(string name, int outlierLimit)
             ? CultureInfo.InvariantCulture.NumberFormat
             : CultureInfo.GetCultureInfo(name).NumberFormat);
 
+        /// <summary>This culture's date separator, when it is one character: what its own dates are written with.</summary>
+        private char DateSeparator { get; } = DateSeparatorOf(name);
+
+        private static char DateSeparatorOf(string name)
+        {
+            string separator = (name.Length == 0 ? CultureInfo.InvariantCulture : CultureInfo.GetCultureInfo(name)).DateTimeFormat.DateSeparator;
+            return separator.Length == 1 ? separator[0] : '\0';
+        }
+
         private int _integer;
         private int _decimal;
         private int _otherSeparator;
+        private int _datesWithOwnSeparator;
         private int _date;
 
         public int NumericCount => _integer + _decimal;
 
         public int DateCount => _date;
 
+        /// <summary>How many of the dates read here are written with this culture's own separator.</summary>
+        public int DatesWithOwnSeparator => _datesWithOwnSeparator;
+
+        /// <summary>An order-free sum over the rows of which date each read as.</summary>
+        public ulong DateFingerprint { get; private set; }
+
+        private static ulong RowDateHash(int rowNumber, DateTime date)
+        {
+            // SplitMix64 over the pair, so that two different readings of a row cannot cancel out.
+            ulong x = ((ulong)(uint)rowNumber << 40) ^ (ulong)date.Ticks;
+            x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9UL;
+            x = (x ^ (x >> 27)) * 0x94D049BB133111EBUL;
+            return x ^ (x >> 31);
+        }
+
         public decimal? MinNumeric { get; private set; }
 
         public decimal? MaxNumeric { get; private set; }
@@ -519,6 +583,16 @@ public void AcceptText(string text, int rowNumber, NumberRead number, bool could
             {
                 _date++;
                 Widen(date);
+
+                if (DateSeparator != '\0' && text.Contains(DateSeparator, StringComparison.Ordinal))
+                {
+                    _datesWithOwnSeparator++;
+                }
+
+                // Which date each row read as, summed so the order of rows does not matter: two
+                // cultures that read every row alike end equal, and one row read otherwise sets them
+                // apart — day and month swapped included.
+                DateFingerprint += RowDateHash(rowNumber, date);
             }
             else if (_dateOutliers.Count < outlierLimit)
             {
@@ -590,6 +664,7 @@ public CultureParseCounts ToCounts() =>
                 Decimal = _decimal,
                 OtherSeparatorDecimals = _otherSeparator,
                 Date = _date,
+                DatesWithOwnSeparator = _datesWithOwnSeparator,
                 NumericOutliers = [.. _numericOutliers],
                 DateOutliers = [.. _dateOutliers],
             };

src/TriasDev.Tabular/Analysis/CultureParseCounts.cs

@@ -21,6 +21,12 @@ public sealed record CultureParseCounts
     /// <summary>Non-empty values that read as a date.</summary>
     public required int Date { get; init; }
 
+    /// <summary>
+    /// How many of <see cref="Date"/> are written with this culture's own date separator — a dot for
+    /// de-DE, a slash for en-US. What breaks a tie between cultures that read every value as a date.
+    /// </summary>
+    public int DatesWithOwnSeparator { get; init; }
+
     /// <summary>
     /// Non-empty values that did not read as a number here only because they write the decimal
     /// separator the other way, and can be read no other way: <c>34.020367</c> under a German reading.

src/TriasDev.Tabular/Analysis/HypothesisBuilder.cs

@@ -60,6 +60,7 @@ public static IReadOnlyList<TypeHypothesis> Build(ColumnFacts facts, double mini
             .OrderBy(h => h.Type == ColumnType.Text ? 1 : 0)
             .ThenByDescending(h => h.Confidence)
             .ThenByDescending(h => Specificity(h.Type))
+            .ThenByDescending(h => OwnSeparatorDates(facts, h))
             .ThenBy(h => h.Culture, StringComparer.Ordinal)];
     }
 
@@ -70,6 +71,15 @@ public static IReadOnlyList<TypeHypothesis> Build(ColumnFacts facts, double mini
     /// Where it did, no culture is involved: the number was a number before anyone read it, and
     /// offering a choice of cultures would invent a question the file already answered.
     /// </remarks>
+    /// <summary>
+    /// For a date reading, how many of its dates are written with its culture's own separator: of two
+    /// readings that fit every value, <c>11.01.2018</c> goes to the one that writes dates with dots (#58).
+    /// </summary>
+    private static int OwnSeparatorDates(ColumnFacts facts, TypeHypothesis hypothesis) =>
+        hypothesis.Type == ColumnType.Date
+            ? facts.ParseCounts.FirstOrDefault(c => c.Culture == hypothesis.Culture)?.DatesWithOwnSeparator ?? 0
+            : 0;
+
     private static bool DeclaredByTheFile(ColumnFacts facts)
     {
         int declared = Native(facts, RawCellKind.Number)

src/TriasDev.Tabular/PublicAPI.Unshipped.txt

@@ -1 +1,5 @@
 #nullable enable
+TriasDev.Tabular.ColumnFacts.DateReadingsDisagree.get -> bool
+TriasDev.Tabular.ColumnFacts.DateReadingsDisagree.init -> void
+TriasDev.Tabular.CultureParseCounts.DatesWithOwnSeparator.get -> int
+TriasDev.Tabular.CultureParseCounts.DatesWithOwnSeparator.init -> void

tests/TriasDev.Tabular.Tests/Analysis/AmbiguousDateTests.cs

@@ -0,0 +1,66 @@
+using System.Text;
+
+using TriasDev.Tabular.Csv;
+
+using Xunit;
+
+namespace TriasDev.Tabular.Tests.Analysis;
+
+/// <summary>
+/// A column whose every day is twelve or less reads as a date under cultures that disagree on which
+/// number is the month. The reading the values' own separator belongs to goes first, and the facts say
+/// that the readings disagree, so nobody takes a coin toss for an answer (#58).
+/// </summary>
+public sealed class AmbiguousDateTests
+{
+    private static CancellationToken Token => TestContext.Current.CancellationToken;
+
+    private static readonly DateTime JanuaryEleventh = new(2018, 1, 11, 0, 0, 0, DateTimeKind.Unspecified);
+
+    private static ColumnProfile Profile(params string[] values)
+    {
+        string csv = "id;signed\n" + string.Concat(values.Select((v, i) => $"{i + 1};{v}\n"));
+        using CsvCursor cursor = new(new MemoryStream(Encoding.UTF8.GetBytes(csv)), "t.csv");
+
+        return TabularAnalyzer.Analyze(cursor, new AnalysisOptions { Cultures = ["", "de-DE", "en-US"] }, cancellationToken: Token).Sheets[0].Columns[1];
+    }
+
+    [Fact]
+    public void PutsTheCultureWhoseSeparatorTheDatesUseFirst()
+    {
+        ColumnProfile column = Profile("11.01.2018", "11.02.2018", "11.03.2018");
+
+        TypeHypothesis best = column.Hypotheses[0];
+        Assert.Equal(ColumnType.Date, best.Type);
+        Assert.Equal("de-DE", best.Culture);
+        Assert.Equal(JanuaryEleventh, column.Facts.MinDate);
+        Assert.Equal(new DateTime(2018, 3, 11, 0, 0, 0, DateTimeKind.Unspecified), column.Facts.MaxDate);
+    }
+
+    [Fact]
+    public void KeepsTheOrderForSlashesWhichTheCulturesShare()
+    {
+        ColumnProfile column = Profile("11/01/2018", "11/02/2018", "11/03/2018");
+
+        Assert.Equal(ColumnType.Date, column.Hypotheses[0].Type);
+        Assert.NotEqual("de-DE", column.Hypotheses[0].Culture);
+    }
+
+    [Theory]
+    [InlineData("11.01.2018", "11.02.2018", "11.03.2018")]
+    [InlineData("11/01/2018", "11/02/2018", "11/03/2018")]
+    [InlineData("01.02.2018", "02.01.2018")]
+    public void SaysTheDateReadingsDisagreeWhenDayAndMonthCanSwap(params string[] values)
+    {
+        Assert.True(Profile(values).Facts.DateReadingsDisagree);
+    }
+
+    [Theory]
+    [InlineData("2018-01-11", "2018-02-11")]
+    [InlineData("25.01.2018", "11.02.2018")]
+    [InlineData("11.11.2018", "12.12.2018")]
+    public void SaysNothingWhereTheReadingsAgreeOrOnlyOneReadsEveryValue(params string[] values)
+    {
+        Assert.False(Profile(values).Facts.DateReadingsDisagree);
+    }
+}