diff --git a/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs b/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs index f18d96b9..4d34e577 100644 --- a/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs +++ b/SW.Bitween.Api/Resources/MappingPreviews/Preview.cs @@ -83,10 +83,10 @@ public async Task Handle(MappingPreviewRequest request) $"Bitween understands up to version {MappingRules.CurrentVersion}.", }; - if (!DocumentFormats.TryGet(rules.SourceFormat, out var source)) + if (!DocumentFormats.TryGet(rules.SourceFormat, out var source, rules.SourceCsv)) return new MappingPreviewResponse { Error = DocumentFormats.Unsupported(rules.SourceFormat, "source") }; - if (!DocumentFormats.TryGet(rules.TargetFormat, out var target)) + if (!DocumentFormats.TryGet(rules.TargetFormat, out var target, rules.TargetCsv)) return new MappingPreviewResponse { Error = DocumentFormats.Unsupported(rules.TargetFormat, "target") }; ValueNode input; diff --git a/SW.Bitween.NativeAdapters/Mapper/DocumentMapper.cs b/SW.Bitween.NativeAdapters/Mapper/DocumentMapper.cs index 0f9e5f8f..1864a05e 100644 --- a/SW.Bitween.NativeAdapters/Mapper/DocumentMapper.cs +++ b/SW.Bitween.NativeAdapters/Mapper/DocumentMapper.cs @@ -98,7 +98,8 @@ private static void MapInto( MappingContext context, SourceTraits traits, List errors, - string path) + string path, + int? written = null) { foreach (var field in fields) { @@ -110,7 +111,7 @@ private static void MapInto( continue; } - if (!TryResolveField(field, scope, context, traits, out var value, out var reason)) + if (!TryResolveField(field, scope, context, traits, written, out var value, out var reason)) { errors.Add(new MappingError(target, reason!)); continue; @@ -134,12 +135,20 @@ private static void MapInto( } /// - /// Builds the list a rule produces: its fixed entries, then one per entry of the source list - /// it walks. + /// Builds the list a rule produces: the entries written before it, one per entry of the source + /// list it walks, then the entries written after. /// /// - /// Fixed entries come first because that is where a header line belongs, and because it is - /// the order the previous mapper produced for the same configuration. + /// + /// The order is the feature. A header line belongs at the top and a trailer at the bottom, and + /// a partner's file routinely ends with a record saying how many came before it. + /// + /// + /// Which is why the source entries are matched before anything is built. The number of rows is + /// then the same wherever is read from — the header, a row, + /// or the trailer — rather than a running total that means something different depending on + /// where it sits. + /// /// private static ListNode BuildList( ListRule rule, @@ -150,19 +159,51 @@ private static ListNode BuildList( string path) { var target = Describe(path, rule.Target); + var rows = Matching(rule, scope, traits, errors, target); var list = ValueNode.List(); - // Read against the scope the list sits in, since a fixed entry has no entry of its own. + // Read against the scope the list sits in, since a written entry has no entry of its own. foreach (var entry in rule.Fixed) - AddEntry(list, entry.Item, entry.Fields, entry.Lists, scope, context, traits, errors, target); + AddEntry(list, entry.Item, entry.Fields, entry.Lists, scope, context, traits, errors, + target, rows.Count); + + foreach (var item in rows) + AddEntry(list, rule.Item, rule.Fields, rule.Lists, scope.Enter(item), context, traits, + errors, target, rows.Count); + + // A trailer belongs on the file whether the source list had a thousand entries, none, or + // was never there at all — a partner expecting a record count still expects to be told + // that it is zero. + foreach (var entry in rule.After) + AddEntry(list, entry.Item, entry.Fields, entry.Lists, scope, context, traits, errors, + target, rows.Count); - // No source list to walk: the list is whatever its fixed entries produced. - if (rule.Over is null) return list; + return list; + } + + /// + /// The entries of the source list that this rule's condition lets through. + /// + /// + /// Separated from building them so that how many there are is known first. Nothing is produced + /// here; a rule that fails does so when its entry is built, as it always did. + /// + private static List Matching( + ListRule rule, + Scope scope, + SourceTraits traits, + List errors, + string target) + { + var matched = new List(); + + // No source list to walk: the list is whatever its written entries produce. + if (rule.Over is null) return matched; // A path that is absent adds nothing rather than failing. An order with no lines is // ordinary; so is an optional section. var over = Values.Resolve(scope.Current, rule.Over); - if (over is null) return list; + if (over is null) return matched; // XML makes a list by repeating a name, so an order with one is the same document as // one whose `line` was never a list. Reading that as no lines would drop the only line @@ -191,10 +232,10 @@ private static ListNode BuildList( break; } - AddEntry(list, rule.Item, rule.Fields, rule.Lists, scope.Enter(item), context, traits, errors, target); + matched.Add(item); } - return list; + return matched; } /// @@ -214,11 +255,12 @@ private static void AddEntry( MappingContext context, SourceTraits traits, List errors, - string target) + string target, + int written) { if (item is not null) { - if (TryResolveField(item, scope, context, traits, out var value, out var reason)) + if (TryResolveField(item, scope, context, traits, written, out var value, out var reason)) list.Add(ValueNode.Value(value)); else errors.Add(new MappingError(target, reason!)); @@ -226,7 +268,7 @@ private static void AddEntry( } var row = ValueNode.Object(); - MapInto(row, fields, lists, scope, context, traits, errors, target); + MapInto(row, fields, lists, scope, context, traits, errors, target, written); list.Add(row); } @@ -235,11 +277,21 @@ private static bool TryResolveField( Scope scope, MappingContext context, SourceTraits traits, + int? written, out object? value, out string? reason) { reason = null; + // Outside a list there is nothing to count, and answering zero would be a number the + // partner would act on rather than a mistake anyone would notice. + if (field.From.Kind == ValueSourceKind.Count && written is null) + { + value = null; + reason = "counting entries only means something inside a list"; + return false; + } + value = field.From.Kind switch { ValueSourceKind.Fixed => field.From.Value, @@ -247,6 +299,7 @@ private static bool TryResolveField( ValueSourceKind.RootPath => Values.ResolveScalar(scope.Root, field.From.Path), ValueSourceKind.Partner => context.PartnerValue(field.From.Key), ValueSourceKind.Global => context.GlobalValue(field.From.SetId, field.From.Key), + ValueSourceKind.Count => (decimal)(written ?? 0), _ => null, }; diff --git a/SW.Bitween.NativeAdapters/Mapper/Formats/CsvFormat.cs b/SW.Bitween.NativeAdapters/Mapper/Formats/CsvFormat.cs new file mode 100644 index 00000000..b936bbfd --- /dev/null +++ b/SW.Bitween.NativeAdapters/Mapper/Formats/CsvFormat.cs @@ -0,0 +1,313 @@ +using System.Globalization; +using System.Text; +using CsvHelper; +using CsvHelper.Configuration; + +namespace SW.Bitween.NativeAdapters.Mapper.Formats; + +/// +/// Reads and writes delimited text — comma, semicolon, pipe or tab. +/// +/// +/// +/// A file in this format is a list of rows, and that is the whole reason it needs so little +/// here. A list rule walking "" already means "the document is itself the list", and +/// Root already means "the whole output is a list" — both written for a JSON document that is +/// a bare array, both already tested. So the mapper is untouched: this is a reader and a writer, and +/// everything between them already exists. +/// +/// +/// Every field is read as text and left alone. Nothing looks at 041800 and decides it is a +/// number, because a tracking reference that loses its leading zero is rejected by the partner and +/// nothing here would ever say why. Where the output wants a real number the rule's own type does +/// it, and it does it because someone asked rather than because something guessed. +/// +/// +/// An object output — not a list — is written as a header and exactly one row. That falls out of +/// treating a row as an object rather than being a case of its own. +/// +/// +public class CsvFormat(CsvOptions? options = null) : IDocumentFormat +{ + private readonly CsvOptions _options = options ?? new CsvOptions(); + + public string Id => "csv"; + + public string ContentType => "text/csv"; + + /// No: a file of one row is a list of one, and says so. + /// + /// Unlike XML, there is no ambiguity to resolve — a row is a row whether there is one of them or + /// a thousand, so nothing has to be tolerated after the fact. + /// + public bool SingleValueIsAList => false; + + /// + /// The separator between a field's name and the name of a field inside it. + /// + /// + /// A row is flat and a mapping is not, so a rule writing to destination.city has to land + /// somewhere. It becomes a column of exactly that name — which is how the editor already shows + /// the path, and the only way to produce such a column at all, since the output-field name box + /// splits what is typed into it on dots. + /// + public const char PathSeparator = '.'; + + public ValueNode Read(string text) + { + if (string.IsNullOrWhiteSpace(text)) + throw new DocumentFormatException("The document is empty, so there is nothing to map."); + + var list = ValueNode.List(); + + try + { + using var reader = new StringReader(StripByteOrderMark(text)); + using var parser = new CsvParser(reader, Configuration(forWriting: false)); + + // One naming run for the whole file, so a column keeps the same name on every row and + // no two columns ever share one. It grows as wider rows turn up. + var names = new List(); + var used = new HashSet(StringComparer.Ordinal); + + if (_options.HasHeader) + { + // A file that is nothing but a header is a list of no rows, which is a true answer + // and not an error — a carrier with nothing to report sends exactly that. + if (!parser.Read()) return list; + foreach (var name in parser.Record ?? []) + names.Add(Unique(name.Length > 0 ? name : Position(names.Count), used)); + } + + while (parser.Read()) + { + var record = parser.Record; + if (record is null || IsBlank(record)) continue; + + // A row wider than anything seen before: the extra fields are named by position + // rather than dropped, which would be the same silent loss as a shared name. + while (names.Count < record.Length) + names.Add(Unique(Position(names.Count), used)); + + var row = ValueNode.Object(); + for (var at = 0; at < record.Length; at++) + row.Set(names[at], ValueNode.Value(record[at])); + + list.Add(row); + } + } + catch (CsvHelperException ex) + { + throw new DocumentFormatException($"The document could not be read as delimited text: {ex.Message}"); + } + + return list; + } + + public string Write(ValueNode root) + { + var rows = root switch + { + ListNode list => list.Items, + // An object is one row. A mapping whose output is a single record has no reason to be + // written as a list of one just to reach this format. + ObjectNode => [root], + _ => throw new DocumentFormatException( + "A delimited file is rows of fields, so the output has to be a list or an object. " + + "This mapping produced a single value."), + }; + + // Every column any row has, in the order they first appear. Rows can legitimately differ — + // entries written into a list carry their own rules — and taking the first row's columns + // would drop the rest without a word. + var columns = new List(); + var seen = new HashSet(StringComparer.Ordinal); + var flattened = new List>(); + + foreach (var row in rows) + { + var cells = new Dictionary(StringComparer.Ordinal); + Flatten(row, prefix: "", cells); + foreach (var column in cells.Keys) + if (seen.Add(column)) + columns.Add(column); + flattened.Add(cells); + } + + var output = new StringWriter(); + using (var writer = new CsvWriter(output, Configuration(forWriting: true))) + { + if (_options.HasHeader) + { + foreach (var column in columns) writer.WriteField(column); + writer.NextRecord(); + } + + foreach (var cells in flattened) + { + // A column this row does not have is written empty rather than skipped: a short row + // would shift every field after it into the wrong column. + foreach (var column in columns) + writer.WriteField(cells.TryGetValue(column, out var value) ? value : ""); + writer.NextRecord(); + } + } + + // Prepended rather than written through the writer, which has no notion of one. Reading + // strips it again, so a file we produce and then read back is unchanged by it. + return _options.ByteOrderMark ? '\ufeff' + output.ToString() : output.ToString(); + } + + /// + /// One row's cells, with nested fields folded into dotted column names. + /// + /// + /// A list inside a row is refused rather than folded. There is no column name that would make + /// ["A1","B7"] fit into one cell, and inventing one — joining them, taking the first — + /// would lose data quietly, which is the one outcome worth refusing over. + /// + private static void Flatten(ValueNode node, string prefix, Dictionary cells) + { + switch (node) + { + case ObjectNode obj: + foreach (var (key, child) in obj.Children()) + Flatten(child, prefix.Length == 0 ? key : prefix + PathSeparator + key, cells); + break; + + // A row that is a single value — a list of plain values, which is a perfectly ordinary + // one-column file of tracking numbers. It has no name to take, so it takes the name any + // column has when nothing names it: its position. + case ScalarNode scalar: + { + var column = prefix.Length == 0 ? Position(0) : prefix; + + // A rule targeting the single key `a.b` and a pair of rules targeting `a` then `b` + // both want the column `a.b`. Only one of them can have it, and quietly keeping + // whichever ran last would drop a field the mapping plainly asks for. + if (!cells.TryAdd(column, AsText(scalar.Value))) + throw new DocumentFormatException( + $"Two rules both write the column '{column}'. A row has one cell per " + + "column, so one of them would be lost. Give one of them another name."); + break; + } + + case ListNode: + throw new DocumentFormatException( + prefix.Length == 0 + ? "A row of a delimited file cannot itself be a list." + : $"'{prefix}' is a list, and a single cell of a delimited file cannot hold " + + "one. Write its entries into named fields instead."); + } + } + + /// + /// The parser and writer settings. + /// + /// + /// Whether to pin the line ending. CsvHelper applies NewLine on reading only when it has + /// been set explicitly, and setting it there would refuse a file that separates its rows with + /// bare newlines — which most of them do. So it is pinned for writing, where RFC 4180 asks for + /// CRLF, and left alone for reading, where whatever the partner sent has to be accepted. + /// + private CsvConfiguration Configuration(bool forWriting) + { + var configuration = new CsvConfiguration(CultureInfo.InvariantCulture) + { + Delimiter = _options.Delimiter, + // Read by hand through the parser rather than mapped onto a class, so CsvHelper is never + // asked to find a header; the names are taken from the first record here instead. + HasHeaderRecord = false, + // A partner file is not a well-formed file. A stray quote in the middle of a field is + // common enough that refusing the whole document over one would stop a day's shipments, + // and the field still arrives — as the characters that are actually there. + BadDataFound = null, + MissingFieldFound = null, + DetectColumnCountChanges = false, + // Whitespace is data. A name written as `s ramanan` and a postcode with a leading space + // both turned up in the first files anyone sent. + TrimOptions = TrimOptions.None, + IgnoreBlankLines = true, + // RFC 4180's set. A field carrying the delimiter, a quote or a line break has to be + // quoted or the file it lands in no longer says what it meant to say. + ShouldQuote = args => + args.Field is not null && + (args.Field.Contains(_options.Delimiter, StringComparison.Ordinal) || + args.Field.Contains('"') || + args.Field.Contains('\r') || + args.Field.Contains('\n')), + }; + + // Assigned only for writing, because assigning it at all is what counts as setting it — + // and a reader pinned to one line ending refuses every file that uses the other. + if (forWriting) configuration.NewLine = "\r\n"; + + return configuration; + } + + /// + /// , or the first name after it that nothing has taken yet. + /// + /// + /// + /// Two columns cannot share a name: a row is built by setting keys on an object, so the second + /// would silently replace the first and a column of the partner's file would simply be missing. + /// + /// + /// Falling back to the position is not enough on its own, because a header can be a number. A + /// file headed 2, names its first column 2 and then wants 2 again for the + /// blank one beside it, so the fallback has to be checked like any other name. + /// + /// + private static string Unique(string wanted, HashSet used) + { + if (used.Add(wanted)) return wanted; + + for (var n = 2; ; n++) + { + var candidate = $"{wanted}_{n}"; + if (used.Add(candidate)) return candidate; + } + } + + /// A field's name when it has none: its position, counting from one. + private static string Position(int at) => (at + 1).ToString(CultureInfo.InvariantCulture); + + /// + /// Whether a record is a blank line rather than a row. + /// + /// + /// IgnoreBlankLines covers an empty line, but a line holding only delimiters — which the + /// samples have between blocks of records — parses as a row of empty fields. Mapping that would + /// produce a row of nothing for every gap in the file. + /// + private static bool IsBlank(string[] record) => record.All(field => field.Length == 0); + + /// + /// Drops the byte-order mark, if the text still carries one. + /// + /// + /// Excel writes one, and left in place it becomes part of the first column's name — so a header + /// of ShipmentNumber arrives as ShipmentNumber and every rule reading it + /// resolves to nothing, with the editor showing a name that looks exactly right. + /// + private static string StripByteOrderMark(string text) => + text.Length > 0 && text[0] == '' ? text[1..] : text; + + /// + /// A value as a field's text. + /// + /// + /// Invariant throughout: a decimal written under a French locale uses a comma, which in a + /// comma-delimited file would silently become an extra column. A decimal keeps the scale it was + /// given, so a weight of 0.100 is written back as 0.100. + /// + private static string AsText(object? value) => value switch + { + null => "", + bool b => b ? "true" : "false", + decimal m => m.ToString(CultureInfo.InvariantCulture), + string s => s, + _ => Convert.ToString(value, CultureInfo.InvariantCulture) ?? "", + }; +} diff --git a/SW.Bitween.NativeAdapters/Mapper/Formats/CsvOptions.cs b/SW.Bitween.NativeAdapters/Mapper/Formats/CsvOptions.cs new file mode 100644 index 00000000..8aa0afc9 --- /dev/null +++ b/SW.Bitween.NativeAdapters/Mapper/Formats/CsvOptions.cs @@ -0,0 +1,53 @@ +namespace SW.Bitween.NativeAdapters.Mapper.Formats; + +/// +/// How one side of a mapping reads or writes delimited text. +/// +/// +/// +/// Kept per side rather than per mapping: a partner's semicolon file is routinely turned into a +/// comma file for somebody else, and one client alone has been seen sending all three of comma, +/// semicolon and pipe. +/// +/// +/// There is no standard worth the name for any of this, which is why none of it is inferred. +/// Sniffing the delimiter from a sample gets it wrong the first time a field legitimately contains +/// a comma, and by then the mapping is in production. +/// +/// +public class CsvOptions +{ + /// The characters between one field and the next. + /// + /// A string rather than a char because a tab travels through JSON as \t, and because + /// multi-character delimiters exist in the wild. + /// + public string Delimiter { get; set; } = ","; + + /// + /// Whether the first line names the columns rather than carrying data. + /// + /// + /// False is not the unusual case. Two of the three files a single client sends start straight + /// into data, so the fields have no names at all and paths are positions instead — see + /// . + /// + public bool HasHeader { get; set; } = true; + + /// + /// Whether to start a written file with a byte-order mark. + /// + /// + /// + /// Three bytes that tell Excel the text is UTF-8. Without them a name like + /// BEAUTRAIT Raphaël opens as Raphaël, and whoever opens it has no way to + /// correct that after the fact — which is why this has to be decided when the file is written. + /// + /// + /// Off by default, because a partner's own parser can just as easily choke on three bytes it + /// did not expect at the start of the file. Reading always strips one, whatever this says: + /// left in place it becomes part of the first column's name. + /// + /// + public bool ByteOrderMark { get; set; } +} diff --git a/SW.Bitween.NativeAdapters/Mapper/Formats/DocumentFormats.cs b/SW.Bitween.NativeAdapters/Mapper/Formats/DocumentFormats.cs index 97792ab3..e0c3340e 100644 --- a/SW.Bitween.NativeAdapters/Mapper/Formats/DocumentFormats.cs +++ b/SW.Bitween.NativeAdapters/Mapper/Formats/DocumentFormats.cs @@ -12,20 +12,39 @@ namespace SW.Bitween.NativeAdapters.Mapper.Formats; /// public static class DocumentFormats { - private static readonly Dictionary ById = + private static readonly JsonFormat Json = new(); + private static readonly XmlFormat Xml = new(); + + /// + /// How each id is built. + /// + /// + /// Factories rather than instances because delimited text is configured per side — a partner's + /// semicolon file routinely becomes somebody else's comma file — so its reader and its writer + /// cannot be the same shared object. The formats that need no configuration stay singletons. + /// + private static readonly Dictionary> ById = new(StringComparer.OrdinalIgnoreCase) { - ["json"] = new JsonFormat(), - ["xml"] = new XmlFormat(), + ["json"] = _ => Json, + ["xml"] = _ => Xml, + ["csv"] = csv => new CsvFormat(csv), }; /// Format ids, for the editor's dropdown and for error messages. public static IReadOnlyList Ids { get; } = ById.Keys.OrderBy(k => k).ToList(); - public static bool TryGet(string? id, [NotNullWhen(true)] out IDocumentFormat? format) + /// The format for an id, configured with where it applies. + public static bool TryGet( + string? id, + [NotNullWhen(true)] out IDocumentFormat? format, + CsvOptions? csv = null) { format = null; - return id is not null && ById.TryGetValue(id, out format); + if (id is null || !ById.TryGetValue(id, out var build)) return false; + + format = build(csv); + return true; } /// diff --git a/SW.Bitween.NativeAdapters/Mapper/MappingRules.cs b/SW.Bitween.NativeAdapters/Mapper/MappingRules.cs index 39c29aef..a22d24fe 100644 --- a/SW.Bitween.NativeAdapters/Mapper/MappingRules.cs +++ b/SW.Bitween.NativeAdapters/Mapper/MappingRules.cs @@ -47,6 +47,17 @@ public class MappingRules /// public DateOrder SourceDateOrder { get; set; } = DateOrder.YearFirst; + /// How the incoming document is delimited, when it is delimited text. + /// + /// Null for every other format, and null here means the defaults. Two options rather than one + /// because the two sides are genuinely independent: reading a partner's semicolon file and + /// writing a comma file for someone else is one mapping. + /// + public Formats.CsvOptions? SourceCsv { get; set; } + + /// How the produced document is delimited, when it is delimited text. + public Formats.CsvOptions? TargetCsv { get; set; } + public List Fields { get; set; } = new(); public List Lists { get; set; } = new(); @@ -143,6 +154,27 @@ public enum ValueSourceKind /// A key in one of the global values sets. Global, + + /// + /// How many entries the source list produced for the list this rule sits in. + /// + /// + /// + /// For a trailer record: a partner's file ends with a line carrying the number of records + /// above it, and nothing else in a mapping knows that number. + /// + /// + /// Entries written by hand are not counted, which is the whole point of the distinction — a + /// trailer says how many records there are, and the header line above it is not one. + /// Counting them would also mean adding a header later quietly changed the trailer, with + /// nothing on screen to say the number had moved. + /// + /// + /// The same number wherever it is read in one list, so a format that puts its count in the + /// header rather than the trailer works without anything special. + /// + /// + Count, } /// How a document writes dates that are not year-first. @@ -261,6 +293,17 @@ public class ListRule /// /// public List Fixed { get; set; } = new(); + + /// + /// Entries put into the list after the walked ones. + /// + /// + /// The other end of , and the reason it exists: a delimited file from a + /// carrier ends with a trailer record carrying the number of records above it. Built exactly + /// the same way, reading against the same scope — the only difference is where they land, and + /// that can see the whole list by the time they run. + /// + public List After { get; set; } = new(); } /// One entry of a list that no source list produced. diff --git a/SW.Bitween.NativeAdapters/Mapper/NativeMapper.cs b/SW.Bitween.NativeAdapters/Mapper/NativeMapper.cs index 8ac31897..15c90b7d 100644 --- a/SW.Bitween.NativeAdapters/Mapper/NativeMapper.cs +++ b/SW.Bitween.NativeAdapters/Mapper/NativeMapper.cs @@ -69,8 +69,8 @@ public void InitializeStartupValues(IDictionary settings) public Task Handle(XchangeFile xchangeFile) { - var source = ResolveFormat(_rules.SourceFormat, "source"); - var target = ResolveFormat(_rules.TargetFormat, "target"); + var source = ResolveFormat(_rules.SourceFormat, "source", _rules.SourceCsv); + var target = ResolveFormat(_rules.TargetFormat, "target", _rules.TargetCsv); var input = source.Read(xchangeFile.Data); var output = DocumentMapper.Map(_rules, input, _context, source.SingleValueIsAList); @@ -136,8 +136,8 @@ private static MappingContext ReadContext(IDictionary settings) } } - private static IDocumentFormat ResolveFormat(string id, string role) => - DocumentFormats.TryGet(id, out var format) + private static IDocumentFormat ResolveFormat(string id, string role, CsvOptions? csv) => + DocumentFormats.TryGet(id, out var format, csv) ? format : throw new InvalidOperationException(DocumentFormats.Unsupported(id, role)); } diff --git a/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj b/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj index 761c8fdb..3f0e3f41 100644 --- a/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj +++ b/SW.Bitween.NativeAdapters/SW.Bitween.NativeAdapters.csproj @@ -15,6 +15,7 @@ + diff --git a/SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs b/SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs new file mode 100644 index 00000000..3a599c73 --- /dev/null +++ b/SW.Bitween.UnitTests/NativeMapper/CsvFormatTests.cs @@ -0,0 +1,528 @@ +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using SW.Bitween.NativeAdapters.Mapper; +using SW.Bitween.NativeAdapters.Mapper.Formats; + +namespace SW.Bitween.UnitTests.NativeMapper; + +/// +/// Reading delimited text into the neutral tree. +/// +/// +/// The three shapes here are the three files a single client actually sends, with nothing +/// smoothed over: a comma file with a header, a pipe file with three record types and no header, +/// and a semicolon file with no header, blank lines between blocks and an accented name in it. +/// Between them they are the reason none of the delimiter, the header or the encoding is inferred. +/// +[TestClass] +public class CsvFormatReadTests +{ + /// Comma, with a header naming seven columns. + private const string Commas = + """ + ShipmentNumber,Reference,TrackingCode,Date,Time,Comment1,Comment2 + 6G61965126082,202493482,SHOR020,2026-09-14,08:29:49,, + 8G49824171336,202340914,SHOR020,2026-09-14,08:34:34,, + """; + + /// Pipe, no header, three record types told apart by the first field. + private const string Pipes = + """ + H|FFSTAT|1|0||||||||||202609141313|1309981|N + D|1309981172|OK|DELIVERY|0.100|KGM|1|||20260908FRACPKT03831|3800351262|202609141307|20260911|NTE|CDG|NTE||BRIAN MATIAS CASTRO PENA|GLOBAL LOGTICS NETWORK|||||||222998693|Clementine Sandri| + D|1309981174|CC|AWAITING CONSIGNEE COLLECTION|0.100|KGM|1|||E824836443|4472825486|202609141305|20260911|MRS|CDG|MRS||CHRISTOPHER GERGES|GLOBAL LOGISTIC NETWORK|||||||222998693|| + T|9|1309981| + """; + + /// Semicolon, no header, blank lines between blocks, an accented name. + private const string Semicolons = + "track;ZT6090386150DE;CDGSF1;SHTW001;20260914;123938;CDG;M+;;;TS;;;BEAUTRAIT Raphaël;ZT6090386150DE;;041800\n" + + "\n" + + "\n" + + "track;ZT6090386205DE;CDGSF1;SHTW001;20260914;123938;CDG;M+;;;TS;;;BEAUTRAIT Raphaël;ZT6090386205DE;;041800\n"; + + private static CsvFormat Format(string delimiter = ",", bool header = true) => + new(new CsvOptions { Delimiter = delimiter, HasHeader = header }); + + /// The rows, as a rule walking the document would see them. + private static ListNode Rows(string text, string delimiter = ",", bool header = true) + { + var read = Format(delimiter, header).Read(text); + return read as ListNode ?? throw new AssertFailedException("a delimited file reads as a list"); + } + + /// The value at of one row, the way a rule reads it. + private static object At(ValueNode row, string path) => Values.ResolveScalar(row, path); + + [TestMethod] + public void A_file_is_a_list_of_rows() + { + // Not an object with a list inside it: the document *is* the list, which is what lets a + // rule walk it with the same empty path a JSON root array already uses. + var read = Format().Read(Commas); + + Assert.IsInstanceOfType(read); + Assert.AreEqual(2, ((ListNode)read).Items.Count, "the header is not a row"); + } + + [TestMethod] + public void A_header_names_the_columns() + { + var rows = Rows(Commas); + + Assert.AreEqual("6G61965126082", At(rows.Items[0], "ShipmentNumber")); + Assert.AreEqual("SHOR020", At(rows.Items[0], "TrackingCode")); + Assert.AreEqual("08:34:34", At(rows.Items[1], "Time")); + } + + [TestMethod] + public void An_empty_field_is_empty_rather_than_missing() + { + // `,,` at the end of every row of the real file. Present and empty is what the file says, + // and it is different from a column that is not there at all. + var rows = Rows(Commas); + + Assert.AreEqual("", At(rows.Items[0], "Comment1")); + Assert.IsNull(At(rows.Items[0], "Comment3"), "a column the file does not have"); + } + + [TestMethod] + public void Without_a_header_the_fields_are_numbered_from_one() + { + var rows = Rows(Pipes, "|", header: false); + + Assert.AreEqual(4, rows.Items.Count, "every line is a row when nothing is a header"); + Assert.AreEqual("H", At(rows.Items[0], "1")); + Assert.AreEqual("D", At(rows.Items[1], "1")); + Assert.AreEqual("1309981172", At(rows.Items[1], "2")); + Assert.AreEqual("BRIAN MATIAS CASTRO PENA", At(rows.Items[1], "18")); + } + + [TestMethod] + public void Rows_of_different_record_types_keep_their_own_lengths() + { + // The reason this matters: the header record has 16 fields, a detail row 28 and the + // trailer 4. Padding them to a common width would invent data; truncating would lose it. + var rows = Rows(Pipes, "|", header: false); + + Assert.AreEqual("N", At(rows.Items[0], "16")); + Assert.IsNull(At(rows.Items[0], "17"), "the header record has no 17th field"); + Assert.AreEqual("1309981", At(rows.Items[3], "3")); + Assert.IsNull(At(rows.Items[3], "5"), "the trailer has four fields"); + } + + [TestMethod] + public void A_leading_zero_survives() + { + // The whole reason nothing is allowed to decide a field looks like a number. 041800 as + // 41800 is a reference the partner rejects, and nothing here would ever say why. + var rows = Rows(Semicolons, ";", header: false); + + Assert.AreEqual("041800", At(rows.Items[0], "17")); + } + + [TestMethod] + public void A_decimal_keeps_the_scale_it_was_written_with() + { + var rows = Rows(Pipes, "|", header: false); + + Assert.AreEqual("0.100", At(rows.Items[1], "5"), "not 0.1"); + } + + [TestMethod] + public void Blank_lines_between_records_are_not_rows() + { + // The semicolon file has runs of them between blocks. Read as rows they would each map to + // an entry of nothing, and a partner would receive a file padded with empty records. + var rows = Rows(Semicolons, ";", header: false); + + Assert.AreEqual(2, rows.Items.Count); + } + + [TestMethod] + public void An_accented_name_is_read_as_written() + { + var rows = Rows(Semicolons, ";", header: false); + + Assert.AreEqual("BEAUTRAIT Raphaël", At(rows.Items[0], "14")); + } + + [TestMethod] + public void A_byte_order_mark_is_not_part_of_the_first_column_name() + { + // Excel writes one. Left in place it becomes part of the first header name, so every rule + // reading that column resolves to nothing while the editor shows a name that looks right. + var rows = Rows("" + Commas); + + Assert.AreEqual("6G61965126082", At(rows.Items[0], "ShipmentNumber")); + } + + [TestMethod] + public void A_value_holding_the_delimiter_is_one_field() + { + var rows = Rows( + """ + sku,address,qty + A1,"Flat 3, Rainbow St",2 + """); + + Assert.AreEqual("Flat 3, Rainbow St", At(rows.Items[0], "address")); + Assert.AreEqual("2", At(rows.Items[0], "qty"), "the quantity has not shifted a column"); + } + + [TestMethod] + public void A_value_holding_a_line_break_is_still_one_row() + { + // Two rows of data written across four lines of text. This is where a reader that splits + // on newlines produces three broken rows and nothing complains. + var rows = Rows("sku,address\nA1,\"Flat 3\nRainbow Street\nAmman\"\n"); + + Assert.AreEqual(1, rows.Items.Count); + Assert.AreEqual("Flat 3\nRainbow Street\nAmman", At(rows.Items[0], "address")); + } + + [TestMethod] + public void A_doubled_quote_is_one_quote() + { + var rows = Rows("sku,note\nA1,\"He said \"\"urgent\"\" twice\"\n"); + + Assert.AreEqual("He said \"urgent\" twice", At(rows.Items[0], "note")); + } + + [TestMethod] + public void Whitespace_inside_a_field_is_kept() + { + // ` s ramanan` and a postcode with a leading space both turned up in real files. Trimming + // is a decision about the partner's data, and not one to take on their behalf. + var rows = Rows("sku,name\nA1, s ramanan \n"); + + Assert.AreEqual(" s ramanan ", At(rows.Items[0], "name")); + } + + [TestMethod] + public void A_repeated_column_name_stays_reachable_under_a_name_of_its_own() + { + // Two columns cannot share a name — a row is built by setting keys on an object, so the + // second would silently replace the first and a column would simply be gone. Numbered + // rather than moved to its position, which reads better and cannot collide with a header + // that happens to be a number. + var rows = Rows("code,code,qty\nA1,B7,2"); + + Assert.AreEqual("A1", At(rows.Items[0], "code")); + Assert.AreEqual("B7", At(rows.Items[0], "code_2")); + Assert.AreEqual("2", At(rows.Items[0], "qty")); + } + + [TestMethod] + public void A_fallback_name_that_a_header_already_took_is_moved_out_of_the_way() + { + // The header's second column has no name, so it wants its position — which the first + // column is already called. Sharing it would mean the second column silently replacing + // the first when the row is built, and a column of the file simply disappearing. + var rows = Rows("2,,qty\na,b,7"); + + Assert.AreEqual("a", At(rows.Items[0], "2")); + Assert.AreEqual("b", At(rows.Items[0], "2_2")); + Assert.AreEqual("7", At(rows.Items[0], "qty")); + } + + [TestMethod] + public void A_header_repeating_a_number_keeps_both_columns() + { + var rows = Rows("2,2\na,b"); + + Assert.AreEqual("a", At(rows.Items[0], "2")); + Assert.AreEqual("b", At(rows.Items[0], "2_2")); + } + + [TestMethod] + public void A_surplus_field_whose_position_is_taken_is_moved_too() + { + // The header names a column `3`, and a row then has a third field wanting the same name. + var rows = Rows("sku,3\nA1,x,surplus"); + + Assert.AreEqual("x", At(rows.Items[0], "3")); + Assert.AreEqual("surplus", At(rows.Items[0], "3_2")); + } + + [TestMethod] + public void Every_row_names_a_column_the_same_way() + { + // Naming runs once for the file rather than per row, so a narrow row followed by a wide + // one cannot end up calling the same column two different things. + var rows = Rows("sku\nA1\nB7,extra"); + + Assert.AreEqual("A1", At(rows.Items[0], "sku")); + Assert.AreEqual("B7", At(rows.Items[1], "sku")); + Assert.AreEqual("extra", At(rows.Items[1], "2")); + } + + [TestMethod] + public void A_field_past_the_end_of_the_header_is_still_readable() + { + var rows = Rows("sku,qty\nA1,2,surprise"); + + Assert.AreEqual("surprise", At(rows.Items[0], "3")); + } + + [TestMethod] + public void A_file_of_nothing_but_a_header_is_no_rows_rather_than_an_error() + { + // A carrier with nothing to report sends exactly this, every morning. + var rows = Rows("ShipmentNumber,Reference\n"); + + Assert.AreEqual(0, rows.Items.Count); + } + + [TestMethod] + public void An_empty_document_is_refused() + { + var thrown = Assert.ThrowsException(() => Format().Read(" ")); + + Assert.IsFalse(string.IsNullOrWhiteSpace(thrown.Message)); + } + + [TestMethod] + public void The_content_type_says_what_it_is() + { + // The previous mapper left this unset and the gateway fell back to application/json, so a + // partner was served a delimited file labelled as JSON. + Assert.AreEqual("text/csv", Format().ContentType); + } +} + +/// +/// Writing the neutral tree back out as delimited text. +/// +[TestClass] +public class CsvFormatWriteTests +{ + private static CsvFormat Format(string delimiter = ",", bool header = true) => + new(new CsvOptions { Delimiter = delimiter, HasHeader = header }); + + /// The document, with newlines normalised so an expectation holds on any platform. + private static string Written(ValueNode tree, string delimiter = ",", bool header = true) => + Format(delimiter, header).Write(tree).Replace("\r\n", "\n"); + + private static ObjectNode Obj(params (string Key, ValueNode Node)[] children) + { + var node = ValueNode.Object(); + foreach (var (key, child) in children) node.Set(key, child); + return node; + } + + private static ValueNode V(object? value) => ValueNode.Value(value); + + private static ListNode L(params ValueNode[] items) + { + var list = ValueNode.List(); + foreach (var item in items) list.Add(item); + return list; + } + + private static string Refuses(ValueNode tree) + { + var thrown = Assert.ThrowsException(() => Format().Write(tree)); + Assert.IsFalse(string.IsNullOrWhiteSpace(thrown.Message), "a refusal must explain itself"); + return thrown.Message; + } + + [TestMethod] + public void A_list_of_records_is_a_header_and_a_row_each() + { + var written = Written(L( + Obj(("sku", V("A1")), ("qty", V(2m))), + Obj(("sku", V("B7")), ("qty", V(5m))))); + + Assert.AreEqual("sku,qty\nA1,2\nB7,5\n", written); + } + + [TestMethod] + public void An_object_is_a_single_row() + { + // A mapping whose output is one record has no reason to be made a list of one just to + // reach this format. + Assert.AreEqual("sku,qty\nA1,2\n", Written(Obj(("sku", V("A1")), ("qty", V(2m))))); + } + + [TestMethod] + public void A_nested_field_becomes_a_dotted_column() + { + // The editor already shows the path this way, and the output-field name box splits what is + // typed into it on dots — so this is the only way such a column can exist at all. + var written = Written(L(Obj( + ("orderId", V("A1")), + ("destination", Obj(("city", V("Amman")), ("country", V("JO"))))))); + + Assert.AreEqual("orderId,destination.city,destination.country\nA1,Amman,JO\n", written); + } + + [TestMethod] + public void Columns_are_every_column_any_row_has() + { + // Rows can differ: entries written into a list carry their own rules. Taking the first + // row's columns would drop the rest without a word. + var written = Written(L( + Obj(("sku", V("A1"))), + Obj(("sku", V("B7")), ("note", V("gift"))))); + + Assert.AreEqual("sku,note\nA1,\nB7,gift\n", written); + } + + [TestMethod] + public void A_column_a_row_lacks_is_written_empty_rather_than_skipped() + { + var written = Written(L( + Obj(("a", V("1")), ("b", V("2")), ("c", V("3"))), + Obj(("a", V("9")), ("c", V("8"))))); + + // Not `9,8`, which would put 8 under b and shift everything after it. + Assert.AreEqual("a,b,c\n1,2,3\n9,,8\n", written); + } + + [TestMethod] + public void A_value_holding_the_delimiter_is_quoted() + { + var written = Written(L(Obj(("sku", V("A1")), ("address", V("Flat 3, Rainbow St"))))); + + Assert.AreEqual("sku,address\nA1,\"Flat 3, Rainbow St\"\n", written); + } + + [TestMethod] + public void A_value_holding_a_quote_has_it_doubled() + { + var written = Written(L(Obj(("note", V("He said \"urgent\" twice"))))); + + Assert.AreEqual("note\n\"He said \"\"urgent\"\" twice\"\n", written); + } + + [TestMethod] + public void A_value_holding_a_line_break_is_quoted() + { + var written = Written(L(Obj(("address", V("Flat 3\nAmman"))))); + + Assert.AreEqual("address\n\"Flat 3\nAmman\"\n", written); + } + + [TestMethod] + public void The_delimiter_is_whatever_was_configured() + { + var tree = L(Obj(("a", V("1")), ("b", V("2")))); + + Assert.AreEqual("a;b\n1;2\n", Written(tree, ";")); + Assert.AreEqual("a|b\n1|2\n", Written(tree, "|")); + Assert.AreEqual("a\tb\n1\t2\n", Written(tree, "\t")); + } + + [TestMethod] + public void Without_a_header_only_the_rows_are_written() + { + var written = Written(L(Obj(("1", V("D")), ("2", V("1309981172")))), "|", header: false); + + Assert.AreEqual("D|1309981172\n", written); + } + + [TestMethod] + public void A_whole_number_has_no_decimal_point() + { + Assert.AreEqual("qty\n2\n", Written(L(Obj(("qty", V(2m)))))); + } + + [TestMethod] + public void A_decimal_keeps_its_scale_and_uses_a_point() + { + // A comma here would become an extra column in a comma-delimited file. + Assert.AreEqual("weight\n0.100\n", Written(L(Obj(("weight", V(0.100m)))))); + } + + [TestMethod] + public void Null_is_an_empty_field() + { + Assert.AreEqual("a,b\n1,\n", Written(L(Obj(("a", V("1")), ("b", V(null)))))); + } + + [TestMethod] + public void A_boolean_is_lower_case() + { + Assert.AreEqual("ok\ntrue\n", Written(L(Obj(("ok", V(true)))))); + } + + [TestMethod] + public void A_list_of_plain_values_is_one_column() + { + // A file of tracking numbers and nothing else. There is no name to take, so the column + // takes the name any column has when nothing names it. + Assert.AreEqual("1\nA1\nB7\n", Written(L(V("A1"), V("B7")))); + } + + [TestMethod] + public void Two_rules_writing_the_same_column_are_refused() + { + // A literal `a.b` key and a nested `a` → `b` both flatten to the column `a.b`. Keeping + // whichever ran last would drop a field the mapping plainly asks for. + var message = Refuses(L(Obj( + ("a.b", V("literal")), + ("a", Obj(("b", V("nested"))))))); + + StringAssert.Contains(message, "a.b"); + } + + [TestMethod] + public void A_list_inside_a_row_is_refused_with_a_reason() + { + var message = Refuses(L(Obj(("sku", V("A1")), ("tags", L(V("cold"), V("fragile")))))); + + StringAssert.Contains(message, "tags"); + } + + [TestMethod] + public void A_single_value_output_is_refused() + { + var message = Refuses(V("just this")); + + StringAssert.Contains(message, "list or an object"); + } +} + +/// +/// Reading a file and writing it back, which is the one test that holds both halves to each other. +/// +[TestClass] +public class CsvRoundTripTests +{ + [TestMethod] + public void A_comma_file_survives_being_read_and_written() + { + const string original = + "ShipmentNumber,Reference,Comment1\r\n" + + "6G61965126082,202493482,\r\n" + + "8G49824171336,202340914,\"a, comment\"\r\n"; + + var format = new CsvFormat(new CsvOptions()); + var written = format.Write(format.Read(original)); + + Assert.AreEqual(original, written); + } + + [TestMethod] + public void A_pipe_file_with_no_header_survives_too() + { + // Ragged on purpose: three records of three different widths, which is the real file. + const string original = + "H|FFSTAT|1\r\n" + + "D|1309981172|OK|DELIVERY\r\n" + + "T|9|1309981\r\n"; + + var format = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }); + var written = format.Write(format.Read(original)); + + // Not identical: every row is written to the widest shape, because a column missing from a + // row has to be written empty or every field after it lands in the wrong column. Reading it + // back gives the same values, which is what a mapping actually depends on. + Assert.AreEqual( + "H|FFSTAT|1|\r\n" + + "D|1309981172|OK|DELIVERY\r\n" + + "T|9|1309981|\r\n", + written); + } +} diff --git a/SW.Bitween.UnitTests/NativeMapper/CsvMappingTests.cs b/SW.Bitween.UnitTests/NativeMapper/CsvMappingTests.cs new file mode 100644 index 00000000..8c06066f --- /dev/null +++ b/SW.Bitween.UnitTests/NativeMapper/CsvMappingTests.cs @@ -0,0 +1,439 @@ +using System.Collections.Generic; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; +using SW.Bitween.NativeAdapters.Mapper; +using SW.Bitween.NativeAdapters.Mapper.Formats; + +namespace SW.Bitween.UnitTests.NativeMapper; + +/// +/// A real client file, mapped end to end. +/// +/// +/// +/// These are the tests that decide whether the design claim holds: that delimited text needs +/// nothing from the engine, because a file is a list of rows and the mapper already walks +/// lists. Nothing here configures a CSV feature — the rules are the same ones a JSON root array +/// uses, and the only thing that changed is which reader produced the tree. +/// +/// +/// The second test is the important one. The pipe file holds three record types in one document, +/// and handling that was supposed to need no new feature: a list's existing "only some entries" +/// filter selects the detail rows and ignores the header and trailer. If that were wrong, the whole +/// shape of the plan would be wrong with it. +/// +/// +[TestClass] +public class CsvMappingTests +{ + /// The client's pipe file: an H record, three D records, a T record. + private const string Tracking = + "H|FFSTAT|1|0||||||||||202609141313|1309981|N\n" + + "D|1309981172|OK|DELIVERY|0.100|KGM|1|||20260908FRACPKT03831|3800351262|202609141307|20260911|NTE|CDG|NTE||BRIAN MATIAS CASTRO PENA|GLOBAL LOGTICS NETWORK|||||||222998693|Clementine Sandri|\n" + + "D|1309981174|CC|AWAITING CONSIGNEE COLLECTION|0.100|KGM|1|||E824836443|4472825486|202609141305|20260911|MRS|CDG|MRS||CHRISTOPHER GERGES|GLOBAL LOGISTIC NETWORK|||||||222998693||\n" + + "D|1309981175|OK|DELIVERY|0.100|KGM|1|||20260908FRACPKT03852|4472869306|202609141309|20260911|GVA|CDG|GVA||AGASH RAMANAN|GLOBAL LOGISTIC NETWORK|||||||222998693|s ramanan|\n" + + "T|9|1309981|\n"; + + /// The client's comma file, which names its columns. + private const string Movements = + "ShipmentNumber,Reference,TrackingCode,Date,Time,Comment1,Comment2\n" + + "6G61965126082,202493482,SHOR020,2026-09-14,08:29:49,,\n" + + "8G49824171336,202340914,SHOR020,2026-09-14,08:34:34,,\n"; + + private static ValueSource Path(string path) => new() { Kind = ValueSourceKind.Path, Path = path }; + + private static FieldRule Field(string target, string path) => + new() { Target = [target], From = Path(path) }; + + private static ValueSource Fixed(object? value) => + new() { Kind = ValueSourceKind.Fixed, Value = value }; + + /// How many entries the enclosing list holds at the point the rule runs. + private static ValueSource Counted() => new() { Kind = ValueSourceKind.Count }; + + private static TransformRule Transform(string fn, params (string Name, object Value)[] args) + { + var rule = new TransformRule { Fn = fn }; + foreach (var (name, value) in args) rule.Args[name] = JToken.FromObject(value); + return rule; + } + + private static string Json(ValueNode tree) => new JsonFormat().Write(tree); + + /// Maps a delimited document with the given rules, and returns the output as JSON. + private static JToken Run(MappingRules rules, string document, CsvOptions source) + { + var reader = new CsvFormat(source); + return JToken.Parse(Json(DocumentMapper.Map( + rules, reader.Read(document), MappingContext.Empty, reader.SingleValueIsAList))); + } + + [TestMethod] + public void A_file_with_a_header_maps_by_column_name() + { + // The whole output is the list, walking the document itself — the same two settings a JSON + // bare array already uses, written before delimited text existed. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Fields = + [ + Field("shipment", "ShipmentNumber"), + Field("reference", "Reference"), + Field("at", "Time"), + ], + }, + }; + + var output = Run(rules, Movements, new CsvOptions()); + + Assert.AreEqual(2, output.Count()); + Assert.AreEqual("6G61965126082", output[0]?["shipment"]?.ToString()); + Assert.AreEqual("202340914", output[1]?["reference"]?.ToString()); + Assert.AreEqual("08:34:34", output[1]?["at"]?.ToString()); + } + + [TestMethod] + public void Record_types_are_separated_by_the_filter_a_list_already_has() + { + // The claim: three record types in one file need no new feature. Field 1 says which kind of + // record a line is, and "only some entries" is exactly the tool for that. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule + { + Field = "1", + Operator = FilterOperator.Equal, + Value = "D", + }, + Fields = + [ + Field("tracking", "2"), + Field("status", "3"), + Field("consignee", "18"), + Field("weight", "5"), + ], + }, + }; + + var output = Run(rules, Tracking, new CsvOptions { Delimiter = "|", HasHeader = false }); + + // Three detail rows. The H record and the T record are gone, and nothing had to know that + // they existed. + Assert.AreEqual(3, output.Count()); + Assert.AreEqual("1309981172", output[0]?["tracking"]?.ToString()); + Assert.AreEqual("CC", output[1]?["status"]?.ToString()); + Assert.AreEqual("AGASH RAMANAN", output[2]?["consignee"]?.ToString()); + + // Still text, still exactly as the partner wrote it. + Assert.AreEqual("0.100", output[0]?["weight"]?.ToString()); + } + + [TestMethod] + public void A_field_can_be_made_a_real_number_where_the_output_wants_one() + { + // Reading leaves every cell as text on purpose. Converting is a decision the rule carries, + // so it happens where someone asked for it and nowhere else. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + Fields = + [ + new FieldRule { Target = ["weight"], From = Path("5"), Type = ValueType.Number }, + // Left alone beside it: a reference that lost its leading characters would be + // rejected by the partner, and this is the same field in the same row. + Field("reference", "10"), + ], + }, + }; + + var output = Run(rules, Tracking, new CsvOptions { Delimiter = "|", HasHeader = false }); + + Assert.AreEqual(JTokenType.Float, output[0]?["weight"]?.Type); + Assert.AreEqual(0.100m, output[0]?["weight"]?.Value()); + Assert.AreEqual("20260908FRACPKT03831", output[0]?["reference"]?.ToString()); + } + + [TestMethod] + public void A_delimited_file_can_be_mapped_into_another_one() + { + // The pipe file becoming the comma file, which is a mapping like any other — one format's + // reader and another's writer, even when they are the same format configured differently. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + Fields = [Field("ShipmentNumber", "11"), Field("TrackingCode", "3")], + }, + }; + + var reader = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }); + var writer = new CsvFormat(new CsvOptions()); + + var written = writer + .Write(DocumentMapper.Map(rules, reader.Read(Tracking), MappingContext.Empty)) + .Replace("\r\n", "\n"); + + Assert.AreEqual( + "ShipmentNumber,TrackingCode\n" + + "3800351262,OK\n" + + "4472825486,CC\n" + + "4472869306,OK\n", + written); + } + + [TestMethod] + public void A_nested_rule_lands_in_a_dotted_column() + { + // Decided rather than discovered: a row is flat, so a nested target has to become a column + // name. The editor already shows the path this way. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + Fields = + [ + new FieldRule { Target = ["destination", "city"], From = Path("16") }, + new FieldRule { Target = ["destination", "hub"], From = Path("15") }, + ], + }, + }; + + var reader = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }); + var written = new CsvFormat(new CsvOptions()) + .Write(DocumentMapper.Map(rules, reader.Read(Tracking), MappingContext.Empty)) + .Replace("\r\n", "\n"); + + StringAssert.StartsWith(written, "destination.city,destination.hub\n"); + StringAssert.Contains(written, "NTE,CDG\n"); + } + + [TestMethod] + public void The_rules_never_mention_a_format_so_the_writer_alone_decides() + { + // What "every pair of formats works" actually rests on. The tree below is built once and + // handed to two different writers; nothing in the rules knows which one is coming. + var fields = new List { Field("tracking", "2"), Field("status", "3") }; + var filter = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }; + var reader = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }); + var source = reader.Read(Tracking); + + var asList = DocumentMapper.Map( + new MappingRules { Root = new ListRule { Over = "", Where = filter, Fields = fields } }, + source, MappingContext.Empty); + + Assert.AreEqual("1309981172", JToken.Parse(Json(asList))[0]?["tracking"]?.ToString()); + StringAssert.Contains(new CsvFormat(new CsvOptions()).Write(asList), "1309981172"); + + // XML is the one target that cannot take a list at the top, because a document has exactly + // one root element and no way to repeat it. That is a fact about XML rather than anything + // to do with where the rows came from, and it is refused by name rather than producing a + // document the partner's parser rejects. + var refused = Assert.ThrowsException( + () => new XmlFormat().Write(asList)); + StringAssert.Contains(refused.Message, "one root element"); + + // Named the list instead, and the same rows reach XML too. + var wrapped = DocumentMapper.Map( + new MappingRules + { + Lists = [new ListRule { Over = "", Where = filter, Fields = fields, Target = ["movements", "movement"] }], + }, + source, MappingContext.Empty); + + StringAssert.Contains(new XmlFormat().Write(wrapped), "1309981172"); + } + + [TestMethod] + public void The_whole_file_can_be_produced_header_rows_and_trailer() + { + // The end of it: reading the carrier's file was never the hard half. This writes one — + // an H line, a D line per shipment, and a T line carrying how many there were. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + // The header line, written before anything is walked. + Fixed = + [ + new ListEntry + { + Fields = + [ + new FieldRule { Target = ["1"], From = Fixed("H") }, + new FieldRule { Target = ["2"], From = Fixed("FFSTAT") }, + ], + }, + ], + Fields = [Field("1", "1"), Field("2", "2"), Field("3", "3")], + // The trailer, written after them — and the only place the count is known. + After = + [ + new ListEntry + { + Fields = + [ + new FieldRule { Target = ["1"], From = Fixed("T") }, + new FieldRule { Target = ["2"], From = Counted() }, + ], + }, + ], + }, + }; + + var reader = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }); + var written = new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }) + .Write(DocumentMapper.Map(rules, reader.Read(Tracking), MappingContext.Empty)) + .Replace("\r\n", "\n"); + + // Three detail rows, so the trailer says 3. The header line is not counted: a trailer + // states how many records there are, and the line announcing the file is not one. + Assert.AreEqual( + "H|FFSTAT|\n" + + "D|1309981172|OK\n" + + "D|1309981174|CC\n" + + "D|1309981175|OK\n" + + "T|3|\n", + written); + } + + [TestMethod] + public void The_count_does_not_move_when_a_header_line_is_added() + { + // The reason it counts rows rather than entries. Building the trailer first and adding + // the header afterwards is the ordinary way round, and the number must not shift under + // it — nothing on screen would say that it had. + var trailer = new ListEntry + { + Fields = [new FieldRule { Target = ["count"], From = Counted() }], + }; + + ListRule ListWith(List before) => new() + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + Fixed = before, + Fields = [Field("1", "1")], + After = [trailer], + }; + + var csv = new CsvOptions { Delimiter = "|", HasHeader = false }; + var withoutHeader = Run(new MappingRules { Root = ListWith([]) }, Tracking, csv); + var withHeader = Run( + new MappingRules + { + Root = ListWith([new ListEntry { Fields = [new FieldRule { Target = ["1"], From = Fixed("H") }] }]), + }, + Tracking, csv); + + Assert.AreEqual(3, withoutHeader.Last()?["count"]?.Value()); + Assert.AreEqual(3, withHeader.Last()?["count"]?.Value(), "adding a header moved nothing"); + } + + [TestMethod] + public void The_count_is_the_same_number_wherever_it_is_read() + { + // So a format that puts its record count in the header rather than the trailer needs + // nothing special, and a row can carry "1 of 3" without the numbers disagreeing. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "D" }, + Fixed = [new ListEntry { Fields = [new FieldRule { Target = ["count"], From = Counted() }] }], + Fields = [new FieldRule { Target = ["count"], From = Counted() }], + After = [new ListEntry { Fields = [new FieldRule { Target = ["count"], From = Counted() }] }], + }, + }; + + var output = Run(rules, Tracking, new CsvOptions { Delimiter = "|", HasHeader = false }); + + foreach (var row in output) + Assert.AreEqual(3, row["count"]?.Value()); + } + + [TestMethod] + public void A_trailer_is_still_written_when_the_source_list_is_empty() + { + // A carrier with nothing to report still sends a file, and the partner still expects to + // be told the count is zero rather than to receive no trailer at all. + var rules = new MappingRules + { + Root = new ListRule + { + Over = "", + Where = new FilterRule { Field = "1", Operator = FilterOperator.Equal, Value = "NONE" }, + Fields = [Field("tracking", "2")], + After = + [ + new ListEntry + { + Fields = [new FieldRule { Target = ["count"], From = Counted() }], + }, + ], + }, + }; + + var output = Run(rules, Tracking, new CsvOptions { Delimiter = "|", HasHeader = false }); + + Assert.AreEqual(1, output.Count(), "the trailer, and nothing else"); + Assert.AreEqual(0, output[0]?["count"]?.Value()); + } + + [TestMethod] + public void Counting_outside_a_list_is_refused_rather_than_answered_with_zero() + { + // Zero is a number a partner would act on. There is no list here, so the honest answer + // is that the rule does not mean anything rather than that the answer is none. + var rules = new MappingRules + { + Fields = [new FieldRule { Target = ["total"], From = Counted() }], + }; + + var thrown = Assert.ThrowsException(() => DocumentMapper.Map( + rules, + new CsvFormat(new CsvOptions { Delimiter = "|", HasHeader = false }).Read(Tracking), + MappingContext.Empty)); + + StringAssert.Contains(thrown.Errors[0].Reason, "inside a list"); + } + + [TestMethod] + public void A_written_file_can_carry_the_mark_Excel_needs() + { + // Without it `BEAUTRAIT Raphaël` opens in Excel as mangled text, and nobody downstream + // can put that right afterwards. + var tree = ValueNode.List(); + var row = ValueNode.Object(); + row.Set("name", ValueNode.Value("BEAUTRAIT Raphaël")); + tree.Add(row); + + var plain = new CsvFormat(new CsvOptions()).Write(tree); + var marked = new CsvFormat(new CsvOptions { ByteOrderMark = true }).Write(tree); + + Assert.IsFalse(plain.StartsWith('\ufeff'), "off unless asked for"); + Assert.IsTrue(marked.StartsWith('\ufeff')); + + // And reading strips it again, so a file we produce and read back is unchanged by it. + Assert.AreEqual( + "BEAUTRAIT Raphaël", + Values.ResolveScalar( + ((ListNode)new CsvFormat(new CsvOptions()).Read(marked)).Items[0], "name")); + } +} diff --git a/SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts b/SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts new file mode 100644 index 00000000..b491f8e0 --- /dev/null +++ b/SW.Bitween.Web/ClientApp/e2e/mapper-csv.spec.ts @@ -0,0 +1,278 @@ +import { test, expect } from "@playwright/test"; +import { signInAsAdmin } from "./helpers"; +import { + addListField, + createSubscription, + expectPreview, + openMapper, + preview, +} from "./mapperHelpers"; + +/** + * Delimited text, through the real editor against the real backend. + * + * The point of running these end to end rather than in a unit test: the tree the editor + * draws comes from d3-dsv in the browser, and the document the mapping actually reads + * comes from CsvHelper on the server. Two parsers, and nothing but a test like this + * notices when they stop agreeing — a column offered here that resolves to nothing there + * is a mapping that looks complete and quietly writes an empty field. + * + * The files are the three a single client really sends, unaltered. + */ + +/** Pipe, no header, three record types told apart by the first field. */ +const TRACKING = + "H|FFSTAT|1|0||||||||||202609141313|1309981|N\n" + + "D|1309981172|OK|DELIVERY|0.100|KGM|1|||20260908FRACPKT03831|3800351262|202609141307|20260911|NTE|CDG|NTE||BRIAN MATIAS CASTRO PENA|GLOBAL LOGTICS NETWORK|||||||222998693|Clementine Sandri|\n" + + "D|1309981174|CC|AWAITING CONSIGNEE COLLECTION|0.100|KGM|1|||E824836443|4472825486|202609141305|20260911|MRS|CDG|MRS||CHRISTOPHER GERGES|GLOBAL LOGISTIC NETWORK|||||||222998693||\n" + + "T|9|1309981|\n"; + +/** Comma, with a header naming its columns. */ +const MOVEMENTS = + "ShipmentNumber,Reference,TrackingCode,Date,Time,Comment1,Comment2\n" + + "6G61965126082,202493482,SHOR020,2026-09-14,08:29:49,,\n" + + "8G49824171336,202340914,SHOR020,2026-09-14,08:34:34,,\n"; + +test.beforeEach(async ({ page }) => { + await signInAsAdmin(page); +}); + +/** Opens the editor with the source side reading delimited text. */ +async function openWithCsv(page: import("@playwright/test").Page, sample: string, delimiter: string, header: boolean) { + const subscriptionId = await createSubscription(page); + await openMapper(page, subscriptionId); + + await page.getByLabel("From format").selectOption("csv"); + await page.getByLabel("source delimiter").selectOption(delimiter); + const box = page.getByRole("checkbox", { name: "source header row" }); + if (header) await box.check(); + else await box.uncheck(); + + await page.getByRole("textbox", { name: "Sample source document" }).fill(sample); + return subscriptionId; +} + +/** Makes the whole output a list walking the document, which is what every row-per-row mapping is. */ +async function rootListOverTheDocument(page: import("@playwright/test").Page) { + await page.getByRole("checkbox", { name: /The whole output is a list/ }).check(); + await page.getByRole("combobox", { name: "Source list" }).selectOption("p:"); + return page.getByRole("group", { name: "Rules for the list at the root" }); +} + +test("a header names the columns, and the server reads the same names", async ({ page }) => { + await openWithCsv(page, MOVEMENTS, ",", true); + + const root = await rootListOverTheDocument(page); + await addListField(root, "the root list", "shipment", "ShipmentNumber"); + await addListField(root, "the root list", "at", "Time"); + + // Produced by the server from its own reading of the file. If the two parsers disagreed + // about where the columns are, this is where it would show. + await expectPreview(page, '"shipment": "6G61965126082"'); + await expect(preview(page)).toContainText('"at": "08:34:34"'); +}); + +test("the columns the editor offers are the ones the server can read", async ({ page }) => { + await openWithCsv(page, MOVEMENTS, ",", true); + + const root = await rootListOverTheDocument(page); + await root.getByRole("button", { name: "Add a field to the root list" }).click(); + await root.getByRole("textbox", { name: "Output field name" }).last().fill("checked"); + + // Every column the source panel offers, tried against the server one at a time — and each + // asserted on the value it should carry. A "not null" check would pass off the previous + // column's preview before the next one arrived, which is exactly the mismatch being hunted. + const field = root.getByRole("combobox", { name: "Source field" }).last(); + const firstRow: [string, string][] = [ + ["ShipmentNumber", "6G61965126082"], + ["Reference", "202493482"], + ["TrackingCode", "SHOR020"], + ["Date", "2026-09-14"], + ["Time", "08:29:49"], + ]; + + for (const [column, value] of firstRow) { + await field.fill(column); + await expect(preview(page)).toContainText(`"checked": "${value}"`, { timeout: 15000 }); + } +}); + +test("three record types in one file are separated by the list's own filter", async ({ page }) => { + await openWithCsv(page, TRACKING, "|", false); + + const root = await rootListOverTheDocument(page); + + // The claim the whole plan rests on: a file holding an H record, D records and a T record + // needs no feature of its own. Field 1 says which kind of line this is. + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + await page.getByRole("checkbox", { name: "Only some entries" }).check(); + await page.getByRole("textbox", { name: "Filter field" }).fill("1"); + await page.getByRole("combobox", { name: "Filter comparison" }).selectOption("equal"); + await page.getByRole("textbox", { name: "Filter value" }).fill("D"); + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + + await addListField(root, "the root list", "tracking", "2"); + await addListField(root, "the root list", "status", "3"); + + await expectPreview(page, '"tracking": "1309981172"'); + await expect(preview(page)).toContainText('"status": "CC"'); + + // The header and the trailer are gone, and nothing had to be told they existed. + await expect(preview(page)).not.toContainText("FFSTAT"); + await expect(preview(page)).not.toContainText('"tracking": "9"'); +}); + +test("a leading zero and a trailing scale survive the round trip to the server", async ({ + page, +}) => { + await openWithCsv(page, TRACKING, "|", false); + + const root = await rootListOverTheDocument(page); + await addListField(root, "the root list", "weight", "5"); + await addListField(root, "the root list", "account", "26"); + + // 0.100 as 0.1, or an account reference losing a character, is a file the partner + // rejects with nothing anywhere to say why. + await expectPreview(page, '"weight": "0.100"'); + await expect(preview(page)).toContainText('"account": "222998693"'); +}); + +test("writing a delimited file takes its delimiter and header from the target side", async ({ + page, +}) => { + await openWithCsv(page, TRACKING, "|", false); + + await page.getByLabel("To format").selectOption("csv"); + await page.getByLabel("target delimiter").selectOption(";"); + await page.getByRole("checkbox", { name: "target header row" }).check(); + + const root = await rootListOverTheDocument(page); + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + await page.getByRole("checkbox", { name: "Only some entries" }).check(); + await page.getByRole("textbox", { name: "Filter field" }).fill("1"); + await page.getByRole("textbox", { name: "Filter value" }).fill("D"); + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + + await addListField(root, "the root list", "Tracking", "2"); + await addListField(root, "the root list", "Status", "3"); + + await expectPreview(page, "Tracking;Status"); + await expect(preview(page)).toContainText("1309981172;OK"); + await expect(preview(page)).toContainText("1309981174;CC"); +}); + +test("a nested rule becomes a dotted column", async ({ page }) => { + await openWithCsv(page, MOVEMENTS, ",", true); + await page.getByLabel("To format").selectOption("csv"); + + const root = await rootListOverTheDocument(page); + // A row is flat, so the nesting has to land somewhere. The name box splits on dots, so + // this is the only way a column of this name can exist at all. + await addListField(root, "the root list", "shipment.number", "ShipmentNumber"); + + await expectPreview(page, "shipment.number"); + await expect(preview(page)).toContainText("6G61965126082"); +}); + +test("a shape a row cannot hold is refused with a reason", async ({ page }) => { + await openWithCsv(page, MOVEMENTS, ",", true); + await page.getByLabel("To format").selectOption("csv"); + + // A list inside a row. There is no cell that holds one, and inventing a way to fit it — + // joining the entries, taking the first — would lose data without a word. + const root = await rootListOverTheDocument(page); + await addListField(root, "the root list", "shipment", "ShipmentNumber"); + await root.getByRole("button", { name: "Add a list to the root list" }).click(); + await root.getByRole("textbox", { name: "Output list name" }).last().fill("tags"); + + await expect(page.getByText(/cannot hold one|cannot itself be a list/)).toBeVisible({ + timeout: 15000, + }); +}); + +/** Adds a field inside one of a list's written entries, pointed at a fixed value. */ +async function addEntryField( + scope: import("@playwright/test").Locator, + entry: string, + name: string, + value: string, +) { + await scope.getByRole("button", { name: `Add a field to ${entry}` }).click(); + await scope.getByRole("textbox", { name: "Output field name" }).last().fill(name); + await scope.getByRole("radio", { name: "Fixed" }).last().click(); + await scope.getByRole("textbox", { name: "Fixed value" }).last().fill(value); +} + +test("the carrier's whole file can be produced, trailer count and all", async ({ page }) => { + // The other direction, and the one that needed two features of its own: a file like the + // client's ends with a record carrying how many records came before it. + await openWithCsv(page, TRACKING, "|", false); + + await page.getByLabel("To format").selectOption("csv"); + await page.getByLabel("target delimiter").selectOption("|"); + await page.getByRole("checkbox", { name: "target header row" }).uncheck(); + + const root = await rootListOverTheDocument(page); + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + await page.getByRole("checkbox", { name: "Only some entries" }).check(); + await page.getByRole("textbox", { name: "Filter field" }).fill("1"); + await page.getByRole("textbox", { name: "Filter value" }).fill("D"); + await page.getByRole("button", { name: "Settings for the list at the root" }).click(); + + // The header line, written before anything is walked. + await root.getByRole("button", { name: "Add an entry to the root list" }).click(); + const first = page.getByRole("group", { name: "Rules for entry 1" }); + await addEntryField(first, "entry 1", "1", "H"); + await addEntryField(first, "entry 1", "2", "FFSTAT"); + + // One line per shipment. + await addListField(root, "the root list", "1", "1"); + await addListField(root, "the root list", "2", "2"); + await addListField(root, "the root list", "3", "3"); + + // And the trailer, which is the only place a rule can see what the list ended up holding. + await root.getByRole("button", { name: "Add a closing entry to the root list" }).click(); + const closing = page.getByRole("group", { name: "Rules for closing entry 1" }); + await addEntryField(closing, "closing entry 1", "1", "T"); + await closing.getByRole("button", { name: "Add a field to closing entry 1" }).click(); + await closing.getByRole("textbox", { name: "Output field name" }).last().fill("2"); + await closing.getByRole("radio", { name: "Count" }).last().click(); + + await expectPreview(page, "H|FFSTAT"); + await expect(preview(page)).toContainText("D|1309981172|OK"); + await expect(preview(page)).toContainText("D|1309981174|CC"); + + // Two shipments, so the trailer says 2 — the header line above it is not a record, and + // adding one later cannot move the number. + await expect(preview(page)).toContainText("T|2"); +}); + +test("counting is offered inside a list and nowhere else", async ({ page }) => { + // Outside a list there is nothing to count, so the segment is not there to be chosen. + await openWithCsv(page, MOVEMENTS, ",", true); + + await page.getByRole("button", { name: "Add a field", exact: true }).click(); + await expect(page.getByRole("radio", { name: "Count" })).toHaveCount(0); + + const root = await rootListOverTheDocument(page); + await root.getByRole("button", { name: "Add a field to the root list" }).click(); + await expect(root.getByRole("radio", { name: "Count" })).toHaveCount(1); +}); + +test("a file can be marked so Excel opens accented names correctly", async ({ page }) => { + await openWithCsv(page, MOVEMENTS, ",", true); + await page.getByLabel("To format").selectOption("csv"); + + const root = await rootListOverTheDocument(page); + await addListField(root, "the root list", "shipment", "ShipmentNumber"); + await expectPreview(page, "shipment"); + + // The mark itself is invisible, so what is checked is that asking for it changes the + // document the server produced rather than that anything looks different. + const before = await preview(page).textContent(); + await page.getByRole("checkbox", { name: "write a byte-order mark" }).check(); + await expect + .poll(async () => (await preview(page).textContent())?.charCodeAt(0), { timeout: 15000 }) + .toBe(0xfeff); + expect(before?.charCodeAt(0)).not.toBe(0xfeff); +}); diff --git a/SW.Bitween.Web/ClientApp/package.json b/SW.Bitween.Web/ClientApp/package.json index c677bb33..2b102b97 100644 --- a/SW.Bitween.Web/ClientApp/package.json +++ b/SW.Bitween.Web/ClientApp/package.json @@ -24,6 +24,7 @@ "@tailwindcss/vite": "^4.3.2", "@tanstack/react-query": "^5.101.2", "@uiw/react-codemirror": "^4.25.11", + "d3-dsv": "^3.0.1", "highlight.js": "^11.12.0", "immer": "^11.1.8", "lucide-react": "^1.24.0", @@ -34,6 +35,7 @@ }, "devDependencies": { "@playwright/test": "^1.61.1", + "@types/d3-dsv": "^3.0.7", "@types/node": "^24.13.2", "@types/react": "^19.2.17", "@types/react-dom": "^19.2.3", diff --git a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/BuildFromSample.tsx b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/BuildFromSample.tsx index 73900b9a..599a7f33 100644 --- a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/BuildFromSample.tsx +++ b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/BuildFromSample.tsx @@ -19,8 +19,8 @@ export function BuildFromSample() { const dispatch = useRulesDispatch(); const parsed = useMemo( - () => parseSample(targetSample, rules.targetFormat, "target"), - [targetSample, rules.targetFormat], + () => parseSample(targetSample, rules.targetFormat, "target", rules.targetCsv), + [targetSample, rules.targetFormat, rules.targetCsv], ); return ( diff --git a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/EntryRow.tsx b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/EntryRow.tsx index 9afaf5f9..b2e3ff91 100644 --- a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/EntryRow.tsx +++ b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/EntryRow.tsx @@ -3,9 +3,9 @@ import { ChevronDown, ChevronRight, CornerDownRight, Trash2 } from "lucide-react import type { OutputEntryNode } from "../../lib/nativeMapper/outputTree"; import { useRules, useRulesDispatch } from "../../lib/nativeMapper/RulesEditorContext"; import { - SOURCE_KINDS, TYPE_BADGES, freshSource, + sourceKindsFor, type EditorFieldRule, } from "../../lib/nativeMapper/types"; import { SegmentedControl } from "../ui/SegmentedControl"; @@ -34,6 +34,8 @@ export function EntryRow({ paths: SourcePaths; }) { const { entry, position } = node; + // Both ends number from one, so the words have to say which end this is. + const named = node.placement === "after" ? `closing entry ${position}` : `entry ${position}`; const dispatch = useRulesDispatch(); const { ruleErrors } = useRules(); const [open, setOpen] = useState(false); @@ -54,16 +56,14 @@ export function EntryRow({ // Named, so this entry's own controls can be told apart from the list's and // from the other entries' — the position is the only thing distinguishing them. role="group" - aria-label={`Entry ${position}`} + aria-label={named} className={`rounded border ${ error ? "border-danger-300 bg-danger-50" : "border-warn-200/70 bg-warn-100/20" }`} >
- - entry {position} - + {named} {item && ( <> @@ -72,8 +72,8 @@ export function EntryRow({ updateItem({ from: freshSource(kind) })} /> @@ -87,7 +87,7 @@ export function EntryRow({ type="button" onClick={() => setOpen((o) => !o)} aria-expanded={open} - aria-label={`Details for entry ${position}`} + aria-label={`Details for ${named}`} title={ extras > 0 ? "Has a transform or a table" @@ -109,7 +109,7 @@ export function EntryRow({
@@ -407,6 +427,75 @@ function TestPartnerSelect({ ); } +/** + * How one side's delimited text is separated, and whether its first line names the columns. + * + * Asked rather than sniffed. A guessed delimiter is right until the first field that + * legitimately contains a comma, and a guessed header is right until a file whose first + * data row happens to look like labels — both of which are found in production rather + * than here. One client alone sends comma, semicolon and pipe, two of the three with no + * header at all, so neither of these has a safe default to fall back on. + */ +function CsvControls({ + side, + options, + onChange, +}: { + side: "source" | "target"; + options: CsvOptions; + onChange: (options: CsvOptions) => void; +}) { + const what = side === "source" ? "incoming" : "produced"; + + return ( +
+ onChange({ ...options, hasHeader: e.target.checked })} + className="size-3.5 rounded border-ink-300" + /> + header + + {/* Only worth asking on the side that writes one. A mark on the way in is stripped + whatever anyone thinks about it, because left in place it becomes part of the first + column's name. */} + {side === "target" && ( + + )} +
+ ); +} + function FormatSelect({ label, value, diff --git a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/OutputRow.tsx b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/OutputRow.tsx index 8c9368f9..a5715170 100644 --- a/SW.Bitween.Web/ClientApp/src/components/nativeMapper/OutputRow.tsx +++ b/SW.Bitween.Web/ClientApp/src/components/nativeMapper/OutputRow.tsx @@ -3,7 +3,7 @@ import { ChevronDown, ChevronRight, Trash2 } from "lucide-react"; import { useRules, useRulesDispatch } from "../../lib/nativeMapper/RulesEditorContext"; import { isAssigned, isItemAssigned } from "../../lib/nativeMapper/rulesReducer"; import type { OutputRowNode } from "../../lib/nativeMapper/outputTree"; -import { SOURCE_KINDS, TYPE_BADGES, freshSource } from "../../lib/nativeMapper/types"; +import { TYPE_BADGES, freshSource, sourceKindsFor } from "../../lib/nativeMapper/types"; import { SegmentedControl } from "../ui/SegmentedControl"; import { RuleDetail } from "./RuleDetail"; import { RowInput } from "./rowControls"; @@ -31,10 +31,13 @@ export function OutputRow({ */ prefix, paths, + /** Whether this rule sits inside a list, which is the only place counting means anything. */ + inList, }: { node: OutputRowNode; prefix: string[]; paths: SourcePaths; + inList: boolean; }) { const { rule, errorKey } = node; const dispatch = useRulesDispatch(); @@ -152,7 +155,7 @@ export function OutputRow({ - +
); } @@ -136,13 +136,15 @@ function TreeNode({ } if (node.kind === "entry") { + const named = + node.placement === "after" ? `closing entry ${node.position}` : `entry ${node.position}`; return ( <>
{node.entry.item === undefined && ( -
+
@@ -207,6 +209,18 @@ function TreeNode({ indent={indent + 1} /> )} + {/* Written after everything above them, which is where a trailer goes — and the + only point from which counting can see the whole list. */} + +
dispatch({ type: "ADD_FIXED_ENTRY", listId: node.list.id }) } + onAddClosingEntry={() => + dispatch({ type: "ADD_CLOSING_ENTRY", listId: node.list.id }) + } showPerEntryRules={node.list.over !== undefined && node.list.item === undefined} onAddValue={ stillUndecided(node.list) @@ -297,6 +314,7 @@ export function AddRuleButtons({ canAddList, inside, onAddFixedEntry, + onAddClosingEntry, onAddValue, showPerEntryRules = true, }: { @@ -306,6 +324,13 @@ export function AddRuleButtons({ inside?: string; /** Offered on a list, where an entry can be written into it. */ onAddFixedEntry?: () => void; + /** + * Offered on a list, for an entry written after the walked ones. + * + * Separate from the leading one because a trailer is a different thing from a header, and + * because it is the only place a rule can count what the list ended up holding. + */ + onAddClosingEntry?: () => void; /** * Offered on a list that has not yet been made of anything. * @@ -344,6 +369,17 @@ export function AddRuleButtons({ Entry )} + {onAddClosingEntry && ( + + )} {showPerEntryRules && (