feat: count and, per binding, import a decimal written with the other separator

docs/importing.md

@@ -161,6 +161,25 @@ defect in the caller and reads as one instead of arriving as an empty column.
 **A row is either values or errors, never both.** Half a row invites half an entity, which is how
 silent corruption starts.
 
+## A column that mixes decimal separators
+
+A hand-assembled file can hold `48,183604` in most rows and `34.020367` in one: a block pasted from
+another tool. Under `de-DE` the second is no number, and since a German group has exactly three digits
+it cannot be one either — its only reading is 34.020367. The profile counts such values per culture,
+as `CultureParseCounts.OtherSeparatorDecimals`, so a screen can say "1 value uses a decimal point"
+rather than show an unexplained outlier.
+
+Importing them is the caller's decision, per binding:
+
+```csharp
+new ColumnBinding { ColumnIndex = 1, FieldName = "lat", Header = "lat", AcceptOtherDecimalSeparator = true }
+```
+
+A decimal field then reads a value written with the other separator, where it can be read no other
+way: digits, that one separator, and not exactly three digits after it. `1.234` stays what the
+culture makes of it. The precheck judges the plan with the setting, and the run counts the values it
+read this way in `ExtractionSummary.OtherSeparatorDecimals`. It is off by default.
+
 ## A field the file says in several languages
 
 A catalogue carries `Title#en` beside `Title#de` — two columns saying one thing. Declared once:

src/TriasDev.Tabular/Analysis/ColumnProfiler.cs

@@ -170,7 +170,7 @@ public void Accept(in RawCell cell, int rowNumber)
             int twin = _numberTwin[i];
 
             _numberReads[i] = twin == i ? _cultures[i].ReadNumber(text, couldBeNumeric) : _numberReads[twin];
-            _cultures[i].AcceptText(text, rowNumber, _numberReads[i], couldBeDate);
+            _cultures[i].AcceptText(text, rowNumber, _numberReads[i], couldBeNumeric, couldBeDate);
         }
     }
 
@@ -385,8 +385,14 @@ private sealed class CultureAccumulator(string name, int outlierLimit)
         private readonly List<ValueLocation> _numericOutliers = [];
         private readonly List<ValueLocation> _dateOutliers = [];
 
+        /// <summary>The decimal separator this culture does not use, looked up once rather than per value.</summary>
+        private char OtherSeparator { get; } = NumberReading.OtherSeparatorOf(name.Length == 0
+            ? CultureInfo.InvariantCulture.NumberFormat
+            : CultureInfo.GetCultureInfo(name).NumberFormat);
+
         private int _integer;
         private int _decimal;
+        private int _otherSeparator;
         private int _date;
 
         public int NumericCount => _integer + _decimal;
@@ -481,7 +487,7 @@ public NumberRead ReadNumber(string text, bool couldBeNumeric)
         /// <param name="rowNumber">Where it stands, for an outlier.</param>
         /// <param name="number">What <see cref="ReadNumber"/> made of it, here or under a twin culture.</param>
         /// <param name="couldBeDate">Whether the value has the shape of a date, asked once by the caller.</param>
-        public void AcceptText(string text, int rowNumber, NumberRead number, bool couldBeDate)
+        public void AcceptText(string text, int rowNumber, NumberRead number, bool couldBeNumeric, bool couldBeDate)
         {
             switch (number.Kind)
             {
@@ -495,8 +501,17 @@ public void AcceptText(string text, int rowNumber, NumberRead number, bool could
                     Widen(number.Value);
                     break;
 
-                case NumberKind.None when _numericOutliers.Count < outlierLimit:
-                    _numericOutliers.Add(new ValueLocation { RowNumber = rowNumber, RawValue = text });
+                case NumberKind.None:
+                    if (couldBeNumeric && NumberReading.IsOtherSeparatorDecimal(text, OtherSeparator))
+                    {
+                        _otherSeparator++;
+                    }
+
+                    if (_numericOutliers.Count < outlierLimit)
+                    {
+                        _numericOutliers.Add(new ValueLocation { RowNumber = rowNumber, RawValue = text });
+                    }
+
                     break;
             }
 
@@ -573,6 +588,7 @@ public CultureParseCounts ToCounts() =>
                 Culture = name,
                 Integer = _integer,
                 Decimal = _decimal,
+                OtherSeparatorDecimals = _otherSeparator,
                 Date = _date,
                 NumericOutliers = [.. _numericOutliers],
                 DateOutliers = [.. _dateOutliers],

src/TriasDev.Tabular/Analysis/CultureParseCounts.cs

@@ -21,6 +21,14 @@ public sealed record CultureParseCounts
     /// <summary>Non-empty values that read as a date.</summary>
     public required int Date { get; init; }
 
+    /// <summary>
+    /// Non-empty values that did not read as a number here only because they write the decimal
+    /// separator the other way, and can be read no other way: <c>34.020367</c> under a German reading.
+    /// They are also among the numeric outliers; a binding may accept them with
+    /// <see cref="ColumnBinding.AcceptOtherDecimalSeparator"/>.
+    /// </summary>
+    public int OtherSeparatorDecimals { get; init; }
+
     /// <summary>Values that did not read as a number, up to the configured limit.</summary>
     public required IReadOnlyList<ValueLocation> NumericOutliers { get; init => field = Equatable.List(value); }
 

src/TriasDev.Tabular/Analysis/NumberReading.cs

@@ -19,6 +19,90 @@ internal static class NumberReading
     /// </remarks>
     public const NumberStyles DecimalStyles = NumberStyles.Number | NumberStyles.AllowExponent;
 
+    /// <summary>
+    /// Reads a value that writes the decimal separator the other way from the culture, where that is
+    /// the only way it can be read: digits, one <c>.</c> or <c>,</c> — whichever the culture does not
+    /// use for decimals — and digits after it that are not exactly three, which would make it a group.
+    /// </summary>
+    /// <remarks>
+    /// <c>34.020367</c> among German decimals cannot be a German grouped number, so its only sensible
+    /// reading is 34.020367 (#47). <c>1.234</c> could be either and is left alone; so is anything
+    /// holding a second separator, a space or an exponent.
+    /// </remarks>
+    public static bool TryReadOtherSeparator(ReadOnlySpan<char> text, NumberFormatInfo format, out decimal value)
+    {
+        value = 0;
+
+        char other = OtherSeparatorOf(format);
+
+        if (!IsOtherSeparatorDecimal(text, other))
+        {
+            return false;
+        }
+
+        ReadOnlySpan<char> trimmed = text.Trim();
+        Span<char> invariant = stackalloc char[trimmed.Length];
+        trimmed.CopyTo(invariant);
+        invariant.Replace(other, '.');
+
+        // Never false for a value the shape admits: at most 28 digits, one point, one sign.
+        return decimal.TryParse(invariant, NumberStyles.AllowLeadingSign | NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out value);
+    }
+
+    /// <summary>The decimal separator a culture does not use — <c>.</c> for <c>,</c> and back — or <c>\0</c>.</summary>
+    public static char OtherSeparatorOf(NumberFormatInfo format) => format.NumberDecimalSeparator switch
+    {
+        "," => '.',
+        "." => ',',
+        _ => '\0',
+    };
+
+    /// <summary>
+    /// Whether the value has the shape <see cref="TryReadOtherSeparator"/> reads, found without
+    /// allocating or parsing: the profile asks it of every value that failed as a number, in every
+    /// culture, and only counts.
+    /// </summary>
+    public static bool IsOtherSeparatorDecimal(ReadOnlySpan<char> text, char other)
+    {
+        ReadOnlySpan<char> body = text.Trim();
+
+        if (other == '\0' || body.Length is 0 or > 30)
+        {
+            return false;
+        }
+
+        if (body[0] is '-' or '+')
+        {
+            body = body[1..];
+        }
+
+        // One pass that gives up at the first character that is neither a digit nor the separator:
+        // the profile asks this of every text value that failed as a number, so a street name must
+        // cost one character, not a scan.
+        int at = -1;
+
+        for (int i = 0; i < body.Length; i++)
+        {
+            char c = body[i];
+
+            if (char.IsAsciiDigit(c))
+            {
+                continue;
+            }
+
+            if (c != other || at >= 0)
+            {
+                return false;
+            }
+
+            at = i;
+        }
+
+        // Digits on both sides, the fraction not exactly three digits long — that would be a group —
+        // and no more than decimal holds.
+        return at > 0 && at < body.Length - 1 && body.Length - at - 1 != 3 && body.Length - 1 <= 28;
+    }
+
     /// <summary>
     /// Whether every group separator in the value is followed by exactly three digits.
     /// </summary>

src/TriasDev.Tabular/Extraction/ExtractionCounters.cs

@@ -17,6 +17,8 @@ internal sealed class ExtractionCounters
 
     public bool StoppedEarly { get; set; }
 
+    public int OtherSeparatorDecimals { get; set; }
+
     public ExtractionSummary Snapshot() => new()
     {
         RowsRead = RowsRead,
@@ -26,5 +28,6 @@ internal sealed class ExtractionCounters
         RowsFailed = RowsFailed,
         ErrorCount = ErrorCount,
         StoppedEarly = StoppedEarly,
+        OtherSeparatorDecimals = OtherSeparatorDecimals,
     };
 }

src/TriasDev.Tabular/Extraction/ExtractionRun.cs

@@ -357,7 +357,7 @@ private void Convert(ReadOnlySpan<RawCell> row)
                 continue;
             }
 
-            if (!ValueReading.TryRead(cell, text, field.Type, _culture, out MappedValue value))
+            if (!TryReadValue(binding, cell, text, field, out MappedValue value))
             {
                 Fail(binding, ErrorCodes.Value.TypeMismatch, text);
                 continue;
@@ -379,6 +379,22 @@ private void Convert(ReadOnlySpan<RawCell> row)
         CheckRequiredGroups();
     }
 
+    /// <summary>Reads a cell as its field's type, counting a decimal read with the other separator.</summary>
+    private bool TryReadValue(ColumnBinding binding, in RawCell cell, string text, ImportField field, out MappedValue value)
+    {
+        if (!ValueReading.TryRead(cell, text, field.Type, _culture, binding.AcceptOtherDecimalSeparator, out value, out bool otherSeparator))
+        {
+            return false;
+        }
+
+        if (otherSeparator)
+        {
+            _counters.OtherSeparatorDecimals++;
+        }
+
+        return true;
+    }
+
     /// <summary>
     /// Fails a row that carries none of a required group's languages.
     /// </summary>

src/TriasDev.Tabular/Extraction/ExtractionSummary.cs

@@ -50,4 +50,10 @@ public sealed record ExtractionSummary
     /// file, so "no further errors" would be a claim nobody checked.
     /// </remarks>
     public bool StoppedEarly { get; init; }
+
+    /// <summary>
+    /// Values a decimal field read with the decimal separator written the other way, because its
+    /// binding accepts them (<see cref="ColumnBinding.AcceptOtherDecimalSeparator"/>).
+    /// </summary>
+    public int OtherSeparatorDecimals { get; init; }
 }

src/TriasDev.Tabular/Import/MappingPrecheck.cs

@@ -315,7 +315,7 @@ void Add(string code, PrecheckSeverity severity, Evidence evidence) =>
         CheckUnique(field, binding, facts, sheet, plan, Add);
         CheckHeader(binding, facts, Add);
         CheckRules(field, binding, facts, culture, Add);
-        CheckType(field, facts, culture, Add);
+        CheckType(field, binding, facts, culture, Add);
     }
 
     /// <summary>Whether a required field's column can supply a value for every row.</summary>
@@ -540,7 +540,7 @@ .. facts.DistinctValues
                 .Where(v => !nothing.Contains(v))
                 .Select(v =>
                 {
-                    bool ok = ValueReading.TryRead(Rebuild(v, declared), v, field.Type, reading, out MappedValue value);
+                    bool ok = ValueReading.TryRead(Rebuild(v, declared), v, field.Type, reading, binding.AcceptOtherDecimalSeparator, out MappedValue value, out _);
                     return (v, value, ok);
                 }),
         ];
@@ -733,6 +733,7 @@ private static bool CannotBeSatisfiedByAnyRow(ImportField field, ColumnBinding b
 
     private static void CheckType(
         ImportField field,
+        ColumnBinding binding,
         ColumnFacts facts,
         string? culture,
         AddFinding add)
@@ -771,7 +772,8 @@ private static void CheckType(
         int readable = field.Type switch
         {
             ColumnType.Integer => counts.Integer,
-            ColumnType.Decimal => counts.Integer + counts.Decimal,
+            // As the import will run it: with the values written the other way, when the binding takes them.
+            ColumnType.Decimal => counts.Integer + counts.Decimal + (binding.AcceptOtherDecimalSeparator ? counts.OtherSeparatorDecimals : 0),
             ColumnType.Date => counts.Date,
             ColumnType.Boolean => facts.BooleanCount,
             _ => facts.NonEmptyCount,

src/TriasDev.Tabular/Mapping/ColumnBinding.cs

@@ -22,4 +22,17 @@ public sealed record ColumnBinding
 
     /// <summary>Values to read as absent, such as a placeholder a spreadsheet uses for "unknown".</summary>
     public IReadOnlyList<string> TreatAsEmpty { get; init => field = Equatable.List(value); } = Equatable.Empty<string>();
+
+    /// <summary>
+    /// Whether a decimal field reads a value that writes the decimal separator the other way from
+    /// the plan's culture, where that is the only way it can be read — <c>34.020367</c> under de-DE.
+    /// </summary>
+    /// <remarks>
+    /// Off by default: the reading is unambiguous, but it is still a guess about a file mixing two
+    /// conventions, and a person decides whether to make it. The profile counts such values per
+    /// culture (<see cref="CultureParseCounts.OtherSeparatorDecimals"/>), the precheck judges the plan
+    /// with this setting, and a run counts the values it read this way
+    /// (<see cref="ExtractionSummary.OtherSeparatorDecimals"/>), so the leniency is never silent.
+    /// </remarks>
+    public bool AcceptOtherDecimalSeparator { get; init; }
 }

src/TriasDev.Tabular/Mapping/ValueReading.cs

@@ -31,6 +31,45 @@ internal static class ValueReading
     /// for it to come out wrong.
     /// </remarks>
     public static bool TryRead(
+        in RawCell cell,
+        string text,
+        ColumnType type,
+        CultureInfo culture,
+        out MappedValue value) =>
+        TryRead(cell, text, type, culture, acceptOtherSeparator: false, out value, out _);
+
+    /// <summary>
+    /// Reads a value as the target field's type, and a decimal written with the other separator too
+    /// when the binding accepts it; says whether it was read that way.
+    /// </summary>
+    public static bool TryRead(
+        in RawCell cell,
+        string text,
+        ColumnType type,
+        CultureInfo culture,
+        bool acceptOtherSeparator,
+        out MappedValue value,
+        out bool readByOtherSeparator)
+    {
+        readByOtherSeparator = false;
+
+        if (TryReadAs(cell, text, type, culture, out value))
+        {
+            return true;
+        }
+
+        if (acceptOtherSeparator && type == ColumnType.Decimal && cell.Kind != RawCellKind.Number
+            && NumberReading.TryReadOtherSeparator(text, culture.NumberFormat, out decimal other))
+        {
+            value = MappedValue.FromDecimal(other);
+            readByOtherSeparator = true;
+            return true;
+        }
+
+        return false;
+    }
+
+    private static bool TryReadAs(
         in RawCell cell,
         string text,
         ColumnType type,

src/TriasDev.Tabular/PublicAPI.Unshipped.txt

@@ -7,3 +7,9 @@ TriasDev.Tabular.SheetVisibility
 TriasDev.Tabular.SheetVisibility.Hidden = 1 -> TriasDev.Tabular.SheetVisibility
 TriasDev.Tabular.SheetVisibility.VeryHidden = 2 -> TriasDev.Tabular.SheetVisibility
 TriasDev.Tabular.SheetVisibility.Visible = 0 -> TriasDev.Tabular.SheetVisibility
+TriasDev.Tabular.ColumnBinding.AcceptOtherDecimalSeparator.get -> bool
+TriasDev.Tabular.ColumnBinding.AcceptOtherDecimalSeparator.init -> void
+TriasDev.Tabular.CultureParseCounts.OtherSeparatorDecimals.get -> int
+TriasDev.Tabular.CultureParseCounts.OtherSeparatorDecimals.init -> void
+TriasDev.Tabular.ExtractionSummary.OtherSeparatorDecimals.get -> int
+TriasDev.Tabular.ExtractionSummary.OtherSeparatorDecimals.init -> void