diff --git a/src/TriasDev.Tabular/Analysis/ColumnFacts.cs b/src/TriasDev.Tabular/Analysis/ColumnFacts.cs index 209d58a..89e56c1 100644 --- a/src/TriasDev.Tabular/Analysis/ColumnFacts.cs +++ b/src/TriasDev.Tabular/Analysis/ColumnFacts.cs @@ -83,6 +83,18 @@ public sealed record ColumnFacts /// public required bool? IsUnique { get; init; } + /// + /// Whether the cultures that read the most values as dates read some row as different dates. + /// + /// + /// True for a column whose every day is twelve or less, written 11.01.2018 or + /// 11/01/2018: German reads it as 11 January, American as 1 November, and every value fits + /// both. The hypotheses rank the culture whose separator the values use first, and the dates in + /// these facts follow it, but that is a preference, not evidence — this says so, so a screen can + /// ask rather than guess (#58). + /// + public bool DateReadingsDisagree { get; init; } + /// /// A bounded sample of values with how often each was seen. /// diff --git a/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs b/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs index 4c473c2..22d8f54 100644 --- a/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs +++ b/src/TriasDev.Tabular/Analysis/ColumnProfiler.cs @@ -208,6 +208,7 @@ public ColumnFacts ToFacts() DistinctCount = _distinctIsExact ? _distinctHashes.Count : _distinctCountAtStop, DistinctCountIsExact = _distinctIsExact, IsUnique = Unique(), + DateReadingsDisagree = DateReadingsDisagree(), DistinctSamples = TopFrequencies(), Samples = [.. _firstValues], DistinctValues = _distinctValuesComplete ? [.. _distinctValues] : [], @@ -228,8 +229,12 @@ private CultureAccumulator BestCulture() foreach (CultureAccumulator culture in _cultures) { + // The third criterion is #58: of cultures reading as many dates, the one whose separator the + // dates are written with, so 11.01.2018 publishes January rather than November. if (culture.NumericCount > best.NumericCount - || (culture.NumericCount == best.NumericCount && culture.DateCount > best.DateCount)) + || (culture.NumericCount == best.NumericCount && culture.DateCount > best.DateCount) + || (culture.NumericCount == best.NumericCount && culture.DateCount == best.DateCount + && culture.DatesWithOwnSeparator > best.DatesWithOwnSeparator)) { best = culture; } @@ -238,6 +243,40 @@ private CultureAccumulator BestCulture() return best; } + /// + /// Whether two cultures that read the most dates read some row as different dates. + /// + private bool DateReadingsDisagree() + { + int most = _cultures.Max(c => c.DateCount); + + if (most == 0) + { + return false; + } + + ulong? first = null; + + foreach (CultureAccumulator culture in _cultures) + { + if (culture.DateCount != most) + { + continue; + } + + if (first is null) + { + first = culture.DateFingerprint; + } + else if (culture.DateFingerprint != first) + { + return true; + } + } + + return false; + } + /// /// Whether this column can identify its rows. /// @@ -390,15 +429,40 @@ private sealed class CultureAccumulator(string name, int outlierLimit) ? CultureInfo.InvariantCulture.NumberFormat : CultureInfo.GetCultureInfo(name).NumberFormat); + /// This culture's date separator, when it is one character: what its own dates are written with. + private char DateSeparator { get; } = DateSeparatorOf(name); + + private static char DateSeparatorOf(string name) + { + string separator = (name.Length == 0 ? CultureInfo.InvariantCulture : CultureInfo.GetCultureInfo(name)).DateTimeFormat.DateSeparator; + return separator.Length == 1 ? separator[0] : '\0'; + } + private int _integer; private int _decimal; private int _otherSeparator; + private int _datesWithOwnSeparator; private int _date; public int NumericCount => _integer + _decimal; public int DateCount => _date; + /// How many of the dates read here are written with this culture's own separator. + public int DatesWithOwnSeparator => _datesWithOwnSeparator; + + /// An order-free sum over the rows of which date each read as. + public ulong DateFingerprint { get; private set; } + + private static ulong RowDateHash(int rowNumber, DateTime date) + { + // SplitMix64 over the pair, so that two different readings of a row cannot cancel out. + ulong x = ((ulong)(uint)rowNumber << 40) ^ (ulong)date.Ticks; + x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9UL; + x = (x ^ (x >> 27)) * 0x94D049BB133111EBUL; + return x ^ (x >> 31); + } + public decimal? MinNumeric { get; private set; } public decimal? MaxNumeric { get; private set; } @@ -519,6 +583,16 @@ public void AcceptText(string text, int rowNumber, NumberRead number, bool could { _date++; Widen(date); + + if (DateSeparator != '\0' && text.Contains(DateSeparator, StringComparison.Ordinal)) + { + _datesWithOwnSeparator++; + } + + // Which date each row read as, summed so the order of rows does not matter: two + // cultures that read every row alike end equal, and one row read otherwise sets them + // apart — day and month swapped included. + DateFingerprint += RowDateHash(rowNumber, date); } else if (_dateOutliers.Count < outlierLimit) { @@ -590,6 +664,7 @@ public CultureParseCounts ToCounts() => Decimal = _decimal, OtherSeparatorDecimals = _otherSeparator, Date = _date, + DatesWithOwnSeparator = _datesWithOwnSeparator, NumericOutliers = [.. _numericOutliers], DateOutliers = [.. _dateOutliers], }; diff --git a/src/TriasDev.Tabular/Analysis/CultureParseCounts.cs b/src/TriasDev.Tabular/Analysis/CultureParseCounts.cs index 752e023..6a0d1b1 100644 --- a/src/TriasDev.Tabular/Analysis/CultureParseCounts.cs +++ b/src/TriasDev.Tabular/Analysis/CultureParseCounts.cs @@ -21,6 +21,12 @@ public sealed record CultureParseCounts /// Non-empty values that read as a date. public required int Date { get; init; } + /// + /// How many of are written with this culture's own date separator — a dot for + /// de-DE, a slash for en-US. What breaks a tie between cultures that read every value as a date. + /// + public int DatesWithOwnSeparator { get; init; } + /// /// Non-empty values that did not read as a number here only because they write the decimal /// separator the other way, and can be read no other way: 34.020367 under a German reading. diff --git a/src/TriasDev.Tabular/Analysis/HypothesisBuilder.cs b/src/TriasDev.Tabular/Analysis/HypothesisBuilder.cs index d38b1c2..223fe8e 100644 --- a/src/TriasDev.Tabular/Analysis/HypothesisBuilder.cs +++ b/src/TriasDev.Tabular/Analysis/HypothesisBuilder.cs @@ -60,6 +60,7 @@ public static IReadOnlyList Build(ColumnFacts facts, double mini .OrderBy(h => h.Type == ColumnType.Text ? 1 : 0) .ThenByDescending(h => h.Confidence) .ThenByDescending(h => Specificity(h.Type)) + .ThenByDescending(h => OwnSeparatorDates(facts, h)) .ThenBy(h => h.Culture, StringComparer.Ordinal)]; } @@ -70,6 +71,15 @@ public static IReadOnlyList Build(ColumnFacts facts, double mini /// Where it did, no culture is involved: the number was a number before anyone read it, and /// offering a choice of cultures would invent a question the file already answered. /// + /// + /// For a date reading, how many of its dates are written with its culture's own separator: of two + /// readings that fit every value, 11.01.2018 goes to the one that writes dates with dots (#58). + /// + private static int OwnSeparatorDates(ColumnFacts facts, TypeHypothesis hypothesis) => + hypothesis.Type == ColumnType.Date + ? facts.ParseCounts.FirstOrDefault(c => c.Culture == hypothesis.Culture)?.DatesWithOwnSeparator ?? 0 + : 0; + private static bool DeclaredByTheFile(ColumnFacts facts) { int declared = Native(facts, RawCellKind.Number) diff --git a/src/TriasDev.Tabular/PublicAPI.Unshipped.txt b/src/TriasDev.Tabular/PublicAPI.Unshipped.txt index 7dc5c58..d74ed17 100644 --- a/src/TriasDev.Tabular/PublicAPI.Unshipped.txt +++ b/src/TriasDev.Tabular/PublicAPI.Unshipped.txt @@ -1 +1,5 @@ #nullable enable +TriasDev.Tabular.ColumnFacts.DateReadingsDisagree.get -> bool +TriasDev.Tabular.ColumnFacts.DateReadingsDisagree.init -> void +TriasDev.Tabular.CultureParseCounts.DatesWithOwnSeparator.get -> int +TriasDev.Tabular.CultureParseCounts.DatesWithOwnSeparator.init -> void diff --git a/tests/TriasDev.Tabular.Tests/Analysis/AmbiguousDateTests.cs b/tests/TriasDev.Tabular.Tests/Analysis/AmbiguousDateTests.cs new file mode 100644 index 0000000..c9d4199 --- /dev/null +++ b/tests/TriasDev.Tabular.Tests/Analysis/AmbiguousDateTests.cs @@ -0,0 +1,66 @@ +using System.Text; + +using TriasDev.Tabular.Csv; + +using Xunit; + +namespace TriasDev.Tabular.Tests.Analysis; + +/// +/// A column whose every day is twelve or less reads as a date under cultures that disagree on which +/// number is the month. The reading the values' own separator belongs to goes first, and the facts say +/// that the readings disagree, so nobody takes a coin toss for an answer (#58). +/// +public sealed class AmbiguousDateTests +{ + private static CancellationToken Token => TestContext.Current.CancellationToken; + + private static readonly DateTime JanuaryEleventh = new(2018, 1, 11, 0, 0, 0, DateTimeKind.Unspecified); + + private static ColumnProfile Profile(params string[] values) + { + string csv = "id;signed\n" + string.Concat(values.Select((v, i) => $"{i + 1};{v}\n")); + using CsvCursor cursor = new(new MemoryStream(Encoding.UTF8.GetBytes(csv)), "t.csv"); + + return TabularAnalyzer.Analyze(cursor, new AnalysisOptions { Cultures = ["", "de-DE", "en-US"] }, cancellationToken: Token).Sheets[0].Columns[1]; + } + + [Fact] + public void PutsTheCultureWhoseSeparatorTheDatesUseFirst() + { + ColumnProfile column = Profile("11.01.2018", "11.02.2018", "11.03.2018"); + + TypeHypothesis best = column.Hypotheses[0]; + Assert.Equal(ColumnType.Date, best.Type); + Assert.Equal("de-DE", best.Culture); + Assert.Equal(JanuaryEleventh, column.Facts.MinDate); + Assert.Equal(new DateTime(2018, 3, 11, 0, 0, 0, DateTimeKind.Unspecified), column.Facts.MaxDate); + } + + [Fact] + public void KeepsTheOrderForSlashesWhichTheCulturesShare() + { + ColumnProfile column = Profile("11/01/2018", "11/02/2018", "11/03/2018"); + + Assert.Equal(ColumnType.Date, column.Hypotheses[0].Type); + Assert.NotEqual("de-DE", column.Hypotheses[0].Culture); + } + + [Theory] + [InlineData("11.01.2018", "11.02.2018", "11.03.2018")] + [InlineData("11/01/2018", "11/02/2018", "11/03/2018")] + [InlineData("01.02.2018", "02.01.2018")] + public void SaysTheDateReadingsDisagreeWhenDayAndMonthCanSwap(params string[] values) + { + Assert.True(Profile(values).Facts.DateReadingsDisagree); + } + + [Theory] + [InlineData("2018-01-11", "2018-02-11")] + [InlineData("25.01.2018", "11.02.2018")] + [InlineData("11.11.2018", "12.12.2018")] + public void SaysNothingWhereTheReadingsAgreeOrOnlyOneReadsEveryValue(params string[] values) + { + Assert.False(Profile(values).Facts.DateReadingsDisagree); + } +}