From e4c0ea6693bdf4fbb6ae132644253d04f5728923 Mon Sep 17 00:00:00 2001 From: Chris Richards Date: Mon, 6 Jul 2026 18:26:37 +0100 Subject: [PATCH 1/3] Parse only the columns a query needs Queries now parse rows in two phases (issue #296): the columns the WHERE clause references are parsed for every candidate row, and the remaining projected (and, when sorting, ORDER BY) columns only once a row matches. The needed-ordinal sets are computed once at bind/prepare time by walking the bound expression tree (SqlColumnCollector); the same treatment applies to DbfQueryDataReader (sequential and index row sources, sync and async), DbfQuery, and both COUNT(*) read-and-filter executors. DbfRecord tracks parsed ordinals with per-row version stamps. Accessing a value that was not parsed for the current row throws a clear InvalidOperationException instead of exposing the previous row's content through the reused value objects. The eager Read path is unaffected - the guard only activates for records the query engine puts in subset mode. Sort buffering also snapshots only the projected and ORDER BY ordinals instead of every column. On the 50,000-row benchmark table, full-scan queries drop 17-21% in time and 72-86% in allocations; selective scans on the 16-column tl_2019 file run 3.6x faster with 11x fewer allocations. Status scans and index-seek paths are unchanged. Benchmark infrastructure fix that this work surfaced: BenchmarkDotNet rebuilds the benchmarks project in a generated child project that does not inherit -p: MSBuild properties, so "-p:DbfDataReaderVersion=x.y.z" silently benchmarked the csproj default. The README now documents the environment-variable form (which reaches every build), and each benchmark prints the informational version of the DbfDataReader assembly it actually loaded. The published benchmarks.md tables were re-verified against correctly-pinned packages and stand as recorded. Closes #296 Co-Authored-By: Claude Fable 5 --- src/DbfDataReader/DbfDataReader.cs | 34 ++ src/DbfDataReader/DbfRecord.cs | 68 +++ src/DbfDataReader/DbfTable.cs | 5 + src/DbfDataReader/Query/CountExecutor.cs | 16 +- src/DbfDataReader/Query/DbfQuery.cs | 60 ++- src/DbfDataReader/Query/DbfQueryDataReader.cs | 56 +- src/DbfDataReader/Query/SqlColumnCollector.cs | 65 +++ .../BenchmarkVersion.cs | 20 + .../DbfDataReaderBenchmarks.cs | 3 + .../IndexQueryBenchmarks.cs | 1 + test/DbfDataReader.Benchmarks/README.md | 13 +- .../QueryColumnSubsetTests.cs | 506 ++++++++++++++++++ 12 files changed, 819 insertions(+), 28 deletions(-) create mode 100644 src/DbfDataReader/Query/SqlColumnCollector.cs create mode 100644 test/DbfDataReader.Benchmarks/BenchmarkVersion.cs create mode 100644 test/DbfDataReader.Tests/QueryColumnSubsetTests.cs diff --git a/src/DbfDataReader/DbfDataReader.cs b/src/DbfDataReader/DbfDataReader.cs index eeee8cb..06d9d81 100644 --- a/src/DbfDataReader/DbfDataReader.cs +++ b/src/DbfDataReader/DbfDataReader.cs @@ -183,6 +183,40 @@ public override async Task ReadAsync(CancellationToken cancellationToken) return result; } + // advances like Read (honoring SkipDeletedRecords) without parsing any column + // values; callers parse the subset they need via DbfRecord.TryParseValues + internal bool ReadRaw() + { + bool result; + bool skip; + do + { + result = DbfTable.ReadRaw(DbfRecord); + if (!result) + break; + + skip = _options.SkipDeletedRecords && DbfRecord.IsDeleted; + } while (skip); + + return result; + } + + internal async ValueTask ReadRawAsync(CancellationToken cancellationToken) + { + bool result; + bool skip; + do + { + result = await DbfTable.ReadRawAsync(DbfRecord, cancellationToken).ConfigureAwait(false); + if (!result) + break; + + skip = _options.SkipDeletedRecords && DbfRecord.IsDeleted; + } while (skip); + + return result; + } + public void Seek(int recordIndex) { DbfTable.Seek(recordIndex); diff --git a/src/DbfDataReader/DbfRecord.cs b/src/DbfDataReader/DbfRecord.cs index 96c98cf..49ad12b 100644 --- a/src/DbfDataReader/DbfRecord.cs +++ b/src/DbfDataReader/DbfRecord.cs @@ -19,6 +19,12 @@ public class DbfRecord private readonly long _dataOffset; private readonly byte[] _buffer; + // column-subset parsing (issue #296): when enabled, an ordinal's value is only + // readable after it has been parsed for the current row; the stamps guard + // against exposing the previous row's content through the reused value objects + private int[] _parsedVersions; + private int _rowVersion; + public DbfRecord(DbfTable dbfTable) { _encoding = dbfTable.CurrentEncoding; @@ -194,6 +200,8 @@ internal async ValueTask ReadRawAsync(Stream stream, CancellationToken can private bool ReadStatus(long position) { + _rowVersion++; + var status = _buffer[0]; if (status == EndOfFile) return false; @@ -212,21 +220,79 @@ private void ParseValues() var slice = span.Slice(dbfValue.Start, dbfValue.Length); dbfValue.Read(slice); } + + MarkAllParsed(); + } + + // restricts value access to ordinals parsed for the current row; used by the + // query engine, which parses only the columns a query references + internal void EnableSubsetParsing() + { + _parsedVersions ??= new int[Values.Count]; + } + + // parses only the given ordinals for the current row; may be called again with + // further ordinals once the row is known to be needed. Returns false when the + // record data extends past the end of a stream (matching Read's contract). + internal bool TryParseValues(int[] ordinals) + { + if (_parsedVersions == null) + throw new InvalidOperationException( + "EnableSubsetParsing must be called before parsing column subsets."); + + try + { + var span = new ReadOnlySpan(_buffer); + foreach (var ordinal in ordinals) + { + var dbfValue = Values[ordinal]; + dbfValue.Read(span.Slice(dbfValue.Start, dbfValue.Length)); + _parsedVersions[ordinal] = _rowVersion; + } + + return true; + } + catch (EndOfStreamException) + { + return false; + } + } + + private void MarkAllParsed() + { + if (_parsedVersions == null) return; + + for (var ordinal = 0; ordinal < _parsedVersions.Length; ordinal++) + { + _parsedVersions[ordinal] = _rowVersion; + } + } + + private void EnsureParsed(int ordinal) + { + if (_parsedVersions != null && _parsedVersions[ordinal] != _rowVersion) + { + throw new InvalidOperationException( + $"The value at ordinal {ordinal} was not parsed for the current row; the query's column subset does not include it."); + } } public object GetValue(int ordinal) { + EnsureParsed(ordinal); var dbfValue = Values[ordinal]; return dbfValue.GetValue(); } public bool IsNull(int ordinal) { + EnsureParsed(ordinal); return Values[ordinal].IsNull; } public T GetValue(int ordinal) { + EnsureParsed(ordinal); var dbfValue = Values[ordinal]; if (dbfValue is DbfValue typedValue) { @@ -242,6 +308,7 @@ public T GetValue(int ordinal) // avoids boxing the value on the way out internal T GetStructValue(int ordinal) where T : struct { + EnsureParsed(ordinal); if (Values[ordinal] is DbfValue typedValue) { var value = typedValue.Value; @@ -275,6 +342,7 @@ private static SqlNullValueException DataIsNull(int ordinal) public string GetStringValue(int ordinal) { + EnsureParsed(ordinal); var dbfValue = Values[ordinal]; try { diff --git a/src/DbfDataReader/DbfTable.cs b/src/DbfDataReader/DbfTable.cs index b7b567a..5e3f200 100644 --- a/src/DbfDataReader/DbfTable.cs +++ b/src/DbfDataReader/DbfTable.cs @@ -207,6 +207,11 @@ internal bool ReadRaw(DbfRecord dbfRecord) return dbfRecord.ReadRaw(Stream); } + internal ValueTask ReadRawAsync(DbfRecord dbfRecord, CancellationToken cancellationToken = default) + { + return dbfRecord.ReadRawAsync(Stream, cancellationToken); + } + public ValueTask ReadAsync(DbfRecord dbfRecord, CancellationToken cancellationToken = default) { return dbfRecord.ReadAsync(Stream, cancellationToken); diff --git a/src/DbfDataReader/Query/CountExecutor.cs b/src/DbfDataReader/Query/CountExecutor.cs index 71d6eac..c13816b 100644 --- a/src/DbfDataReader/Query/CountExecutor.cs +++ b/src/DbfDataReader/Query/CountExecutor.cs @@ -48,7 +48,10 @@ public static (int Count, string Description) Execute(SelectStatement statement, } var evaluator = new SqlExpressionEvaluator(statement.Where, namedParameters, positionalParameters); - return (CountByReadingRows(reader, evaluator, plan), description); + var filterOrdinals = + SqlColumnCollector.ToSortedOrdinals(SqlColumnCollector.CollectOrdinals(statement.Where)); + record.EnableSubsetParsing(); + return (CountByReadingRows(reader, evaluator, plan, filterOrdinals), description); } private static int CountByStatusScan(DbfTable table, DbfRecord record, bool skipDeletedRecords) @@ -79,8 +82,9 @@ private static int CountByStatusChecks(DbfTable table, DbfRecord record, IReadOn return count; } + // rows are counted parsing only the columns the WHERE clause references private static int CountByReadingRows(DbfDataReader reader, SqlExpressionEvaluator evaluator, - QueryAccessPlan plan) + QueryAccessPlan plan, int[] filterOrdinals) { Func accessor = reader.GetValue; var table = reader.DbfTable; @@ -90,9 +94,10 @@ private static int CountByReadingRows(DbfDataReader reader, SqlExpressionEvaluat if (plan.RecordIndexes == null) { - // reader.Read applies the skip-deleted option itself - while (reader.Read()) + // reader.ReadRaw applies the skip-deleted option itself + while (reader.ReadRaw()) { + if (!record.TryParseValues(filterOrdinals)) break; if (evaluator.Matches(accessor)) count++; } @@ -102,8 +107,9 @@ private static int CountByReadingRows(DbfDataReader reader, SqlExpressionEvaluat foreach (var recordIndex in plan.RecordIndexes) { table.Seek(recordIndex); - if (!table.Read(record)) continue; + if (!table.ReadRaw(record)) continue; if (reader.SkipsDeletedRecords && record.IsDeleted) continue; + if (!record.TryParseValues(filterOrdinals)) break; if (!evaluator.Matches(accessor)) continue; count++; diff --git a/src/DbfDataReader/Query/DbfQuery.cs b/src/DbfDataReader/Query/DbfQuery.cs index d3cfbc0..63c9e17 100644 --- a/src/DbfDataReader/Query/DbfQuery.cs +++ b/src/DbfDataReader/Query/DbfQuery.cs @@ -130,6 +130,7 @@ public int Count() { var plan = Prepare(); var record = new DbfRecord(_table); + record.EnableSubsetParsing(); _table.Seek(0); @@ -190,7 +191,9 @@ private int CountByReadingRows(QueryPlan plan, DbfRecord record, int limit) var count = 0; while (count < limit && ReadNextRow(plan, record, ref position)) { - if (!Accept(record, plan, accessor)) continue; + if (!_includeDeleted && record.IsDeleted) continue; + if (!record.TryParseValues(plan.FilterOrdinals)) break; + if (plan.Filter != null && !plan.Filter.Matches(accessor)) continue; count++; } @@ -203,13 +206,13 @@ private int CountByReadingRows(QueryPlan plan, DbfRecord record, int limit) private bool ReadNextRow(QueryPlan plan, DbfRecord record, ref int position) { var recordIndexes = plan.AccessPlan.RecordIndexes; - if (recordIndexes == null) return _table.Read(record); + if (recordIndexes == null) return _table.ReadRaw(record); while (position < recordIndexes.Count) { _table.Seek(recordIndexes[position]); position++; - if (_table.Read(record)) return true; // false: entry beyond the table + if (_table.ReadRaw(record)) return true; // false: entry beyond the table } return false; @@ -231,6 +234,7 @@ public async IAsyncEnumerable AsAsyncEnumerable( { var plan = Prepare(); var record = new DbfRecord(_table); + record.EnableSubsetParsing(); Func accessor = record.GetValue; var cursor = new RowCursor(); @@ -242,7 +246,8 @@ public async IAsyncEnumerable AsAsyncEnumerable( while (NotLimited(returned) && await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(false)) { - if (!Accept(record, plan, accessor)) continue; + if (!MatchesFilter(record, plan, accessor)) continue; + if (!record.TryParseValues(plan.PostFilterOrdinals)) break; yield return Materialize(record, plan); returned++; @@ -254,7 +259,8 @@ await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(f var buffer = new List<(T Item, object[] Keys)>(); while (await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(false)) { - if (!Accept(record, plan, accessor)) continue; + if (!MatchesFilter(record, plan, accessor)) continue; + if (!record.TryParseValues(plan.PostFilterOrdinals)) break; buffer.Add((Materialize(record, plan), SnapshotSortKeys(record, plan))); } @@ -275,13 +281,13 @@ private async Task ReadNextRowAsync(QueryPlan plan, DbfRecord record, RowC { var recordIndexes = plan.AccessPlan.RecordIndexes; if (recordIndexes == null) - return await _table.ReadAsync(record, cancellationToken).ConfigureAwait(false); + return await _table.ReadRawAsync(record, cancellationToken).ConfigureAwait(false); while (cursor.Position < recordIndexes.Count) { _table.Seek(recordIndexes[cursor.Position]); cursor.Position++; - if (await _table.ReadAsync(record, cancellationToken).ConfigureAwait(false)) return true; + if (await _table.ReadRawAsync(record, cancellationToken).ConfigureAwait(false)) return true; } return false; @@ -291,6 +297,7 @@ private IEnumerable Execute() { var plan = Prepare(); var record = new DbfRecord(_table); + record.EnableSubsetParsing(); Func accessor = record.GetValue; _table.Seek(0); @@ -301,7 +308,8 @@ private IEnumerable Execute() var returned = 0; while (NotLimited(returned) && ReadNextRow(plan, record, ref position)) { - if (!Accept(record, plan, accessor)) continue; + if (!MatchesFilter(record, plan, accessor)) continue; + if (!record.TryParseValues(plan.PostFilterOrdinals)) break; yield return Materialize(record, plan); returned++; @@ -313,7 +321,8 @@ private IEnumerable Execute() var buffer = new List<(T Item, object[] Keys)>(); while (ReadNextRow(plan, record, ref position)) { - if (!Accept(record, plan, accessor)) continue; + if (!MatchesFilter(record, plan, accessor)) continue; + if (!record.TryParseValues(plan.PostFilterOrdinals)) break; buffer.Add((Materialize(record, plan), SnapshotSortKeys(record, plan))); } @@ -334,9 +343,10 @@ private bool NotLimited(int returned) return _take == null || returned < _take.Value; } - private bool Accept(DbfRecord record, QueryPlan plan, Func accessor) + private bool MatchesFilter(DbfRecord record, QueryPlan plan, Func accessor) { if (!_includeDeleted && record.IsDeleted) return false; + if (!record.TryParseValues(plan.FilterOrdinals)) return false; return plan.Filter == null || plan.Filter.Matches(accessor); } @@ -457,20 +467,40 @@ int ResolveOrdinal(string propertyName) var accessPlan = QueryPlanner.CreatePlan(combinedWhere, sortKeys, _table, _useIndexes, null, null); + // column-subset parsing (issue #296): the filter's columns are parsed for + // every candidate row; the remaining mapped (and, when sorting, sort key) + // columns only for rows that match + var filterOrdinals = combinedWhere == null + ? new HashSet() + : SqlColumnCollector.CollectOrdinals(combinedWhere); + + var needed = new HashSet(mappedOrdinals); + if (sortKeys.Count > 0 && !accessPlan.SortSatisfied) + { + foreach (var (ordinal, _) in sortKeys) + { + needed.Add(ordinal); + } + } + + needed.ExceptWith(filterOrdinals); + return new QueryPlan( RowMaterializer.Create(mappedNames), mappedOrdinals, new object[mappedOrdinals.Count], filter, sortKeys, - accessPlan); + accessPlan, + SqlColumnCollector.ToSortedOrdinals(filterOrdinals), + SqlColumnCollector.ToSortedOrdinals(needed)); } private sealed class QueryPlan { public QueryPlan(Func materializer, IReadOnlyList mappedOrdinals, object[] rowBuffer, SqlExpressionEvaluator filter, IReadOnlyList<(int Ordinal, bool Descending)> sortKeys, - QueryAccessPlan accessPlan) + QueryAccessPlan accessPlan, int[] filterOrdinals, int[] postFilterOrdinals) { Materializer = materializer; MappedOrdinals = mappedOrdinals; @@ -478,6 +508,8 @@ public QueryPlan(Func materializer, IReadOnlyList mappedOrdina Filter = filter; SortKeys = sortKeys; AccessPlan = accessPlan; + FilterOrdinals = filterOrdinals; + PostFilterOrdinals = postFilterOrdinals; } public Func Materializer { get; } @@ -491,6 +523,10 @@ public QueryPlan(Func materializer, IReadOnlyList mappedOrdina public IReadOnlyList<(int Ordinal, bool Descending)> SortKeys { get; } public QueryAccessPlan AccessPlan { get; } + + public int[] FilterOrdinals { get; } + + public int[] PostFilterOrdinals { get; } } } } diff --git a/src/DbfDataReader/Query/DbfQueryDataReader.cs b/src/DbfDataReader/Query/DbfQueryDataReader.cs index 054ee59..6b0d30a 100644 --- a/src/DbfDataReader/Query/DbfQueryDataReader.cs +++ b/src/DbfDataReader/Query/DbfQueryDataReader.cs @@ -25,6 +25,9 @@ internal sealed class DbfQueryDataReader : DbDataReader, IDbColumnSchemaGenerato private readonly QueryAccessPlan _plan; private readonly bool _skipDeletedRecords; private readonly int _limit; // -1 when unlimited + private readonly int[] _filterParseOrdinals; // parsed before the filter runs + private readonly int[] _postFilterParseOrdinals; // parsed only for matching rows + private readonly int[] _snapshotOrdinals; // ordinals captured into sort-buffer rows private int _rowsReturned; private int _planPosition; // cursor into the plan's record indexes @@ -70,6 +73,30 @@ public DbfQueryDataReader(DbfDataReader reader, SelectStatement statement, SqlEx _ordinals = ordinals; _names = names; _limit = statement.Top ?? -1; + + // column-subset parsing (issue #296): rows are parsed in two phases - the + // columns the filter references first, then, only once a row matches, the + // remaining projected (and, when sorting, ORDER BY) columns + var filterOrdinals = statement.Where == null + ? new HashSet() + : SqlColumnCollector.CollectOrdinals(statement.Where); + + var needed = new HashSet(ordinals); + if (SortRequired) + { + foreach (var item in _orderBy) + { + needed.Add(item.Ordinal); + } + } + + _snapshotOrdinals = SqlColumnCollector.ToSortedOrdinals(needed); + + needed.ExceptWith(filterOrdinals); + _filterParseOrdinals = SqlColumnCollector.ToSortedOrdinals(filterOrdinals); + _postFilterParseOrdinals = SqlColumnCollector.ToSortedOrdinals(needed); + + reader.DbfRecord.EnableSubsetParsing(); } public override int FieldCount => _names.Count; @@ -104,6 +131,7 @@ public override bool Read() while (ReadNextRow()) { if (_filter != null && !_filter.Matches(_readerValueAccessor)) continue; + if (!_reader.DbfRecord.TryParseValues(_postFilterParseOrdinals)) return false; _rowsReturned++; return true; @@ -125,6 +153,7 @@ public override async Task ReadAsync(CancellationToken cancellationToken) while (await ReadNextRowAsync(cancellationToken).ConfigureAwait(false)) { if (_filter != null && !_filter.Matches(_readerValueAccessor)) continue; + if (!_reader.DbfRecord.TryParseValues(_postFilterParseOrdinals)) return false; _rowsReturned++; return true; @@ -133,12 +162,14 @@ public override async Task ReadAsync(CancellationToken cancellationToken) return false; } - // advances the underlying reader to the next candidate row: sequentially, or by + // advances the underlying reader to the next candidate row - sequentially, or by // seeking the next record index supplied by the access plan (re-checking the - // deleted flag, which sequential reads handle inside the raw reader) + // deleted flag, which sequential reads handle inside the raw reader) - parsing + // only the columns the filter needs private bool ReadNextRow() { - if (_plan?.RecordIndexes == null) return _reader.Read(); + if (_plan?.RecordIndexes == null) + return _reader.ReadRaw() && _reader.DbfRecord.TryParseValues(_filterParseOrdinals); while (_planPosition < _plan.RecordIndexes.Count) { @@ -146,10 +177,10 @@ private bool ReadNextRow() _planPosition++; _reader.Seek(recordIndex); - if (!_reader.DbfTable.Read(_reader.DbfRecord)) continue; // entry beyond the table + if (!_reader.DbfTable.ReadRaw(_reader.DbfRecord)) continue; // entry beyond the table if (_skipDeletedRecords && _reader.DbfRecord.IsDeleted) continue; - return true; + return _reader.DbfRecord.TryParseValues(_filterParseOrdinals); } return false; @@ -158,7 +189,10 @@ private bool ReadNextRow() private async Task ReadNextRowAsync(CancellationToken cancellationToken) { if (_plan?.RecordIndexes == null) - return await _reader.ReadAsync(cancellationToken).ConfigureAwait(false); + { + return await _reader.ReadRawAsync(cancellationToken).ConfigureAwait(false) && + _reader.DbfRecord.TryParseValues(_filterParseOrdinals); + } while (_planPosition < _plan.RecordIndexes.Count) { @@ -166,11 +200,11 @@ private async Task ReadNextRowAsync(CancellationToken cancellationToken) _planPosition++; _reader.Seek(recordIndex); - if (!await _reader.DbfTable.ReadAsync(_reader.DbfRecord, cancellationToken).ConfigureAwait(false)) + if (!await _reader.DbfTable.ReadRawAsync(_reader.DbfRecord, cancellationToken).ConfigureAwait(false)) continue; if (_skipDeletedRecords && _reader.DbfRecord.IsDeleted) continue; - return true; + return _reader.DbfRecord.TryParseValues(_filterParseOrdinals); } return false; @@ -192,6 +226,7 @@ private List BuildSortedRows() while (ReadNextRow()) { if (_filter != null && !_filter.Matches(_readerValueAccessor)) continue; + if (!_reader.DbfRecord.TryParseValues(_postFilterParseOrdinals)) break; rows.Add(SnapshotCurrentRow()); } @@ -206,6 +241,7 @@ private async Task> BuildSortedRowsAsync(CancellationToken cancel while (await ReadNextRowAsync(cancellationToken).ConfigureAwait(false)) { if (_filter != null && !_filter.Matches(_readerValueAccessor)) continue; + if (!_reader.DbfRecord.TryParseValues(_postFilterParseOrdinals)) break; rows.Add(SnapshotCurrentRow()); } @@ -214,10 +250,12 @@ private async Task> BuildSortedRowsAsync(CancellationToken cancel return rows; } + // full-width so underlying ordinals index it directly; only the projected and + // ORDER BY positions are populated (and only those are ever read back) private object[] SnapshotCurrentRow() { var row = new object[_reader.FieldCount]; - for (var ordinal = 0; ordinal < row.Length; ordinal++) + foreach (var ordinal in _snapshotOrdinals) { row[ordinal] = _reader.GetValue(ordinal); } diff --git a/src/DbfDataReader/Query/SqlColumnCollector.cs b/src/DbfDataReader/Query/SqlColumnCollector.cs new file mode 100644 index 0000000..9471a61 --- /dev/null +++ b/src/DbfDataReader/Query/SqlColumnCollector.cs @@ -0,0 +1,65 @@ +using System.Collections.Generic; + +namespace DbfDataReader.Query +{ + // collects the ordinals of every column a bound expression references, so the + // engine can parse only the values a query actually needs (issue #296) + internal static class SqlColumnCollector + { + public static HashSet CollectOrdinals(SqlExpression expression) + { + var ordinals = new HashSet(); + Collect(expression, ordinals); + return ordinals; + } + + public static int[] ToSortedOrdinals(HashSet ordinals) + { + var result = new int[ordinals.Count]; + ordinals.CopyTo(result); + System.Array.Sort(result); + return result; + } + + private static void Collect(SqlExpression expression, HashSet ordinals) + { + switch (expression) + { + case null: + return; + case SqlColumnExpression column: + ordinals.Add(column.Ordinal); + return; + case SqlBinaryExpression binary: + Collect(binary.Left, ordinals); + Collect(binary.Right, ordinals); + return; + case SqlNotExpression not: + Collect(not.Operand, ordinals); + return; + case SqlBetweenExpression between: + Collect(between.Operand, ordinals); + Collect(between.Low, ordinals); + Collect(between.High, ordinals); + return; + case SqlInExpression inExpression: + Collect(inExpression.Operand, ordinals); + foreach (var value in inExpression.Values) + { + Collect(value, ordinals); + } + + return; + case SqlLikeExpression like: + Collect(like.Operand, ordinals); + Collect(like.Pattern, ordinals); + return; + case SqlIsNullExpression isNull: + Collect(isNull.Operand, ordinals); + return; + default: + return; // literals and parameters reference no columns + } + } + } +} diff --git a/test/DbfDataReader.Benchmarks/BenchmarkVersion.cs b/test/DbfDataReader.Benchmarks/BenchmarkVersion.cs new file mode 100644 index 0000000..e09b94c --- /dev/null +++ b/test/DbfDataReader.Benchmarks/BenchmarkVersion.cs @@ -0,0 +1,20 @@ +using System; +using System.Reflection; + +namespace DbfDataReader.Benchmarks +{ + // BenchmarkDotNet rebuilds this project in a generated child project, which does + // NOT inherit -p:DbfDataReaderVersion from the command line (only the environment + // variable form reaches it - see the README). Printing the version the benchmark + // process actually loaded makes a silently wrong package impossible to miss. + internal static class BenchmarkVersion + { + public static void Print() + { + var assembly = typeof(DbfDataReader).Assembly; + var version = assembly.GetCustomAttribute() + ?.InformationalVersion ?? assembly.GetName().Version?.ToString(); + Console.WriteLine($"// DbfDataReader under benchmark: {version}"); + } + } +} diff --git a/test/DbfDataReader.Benchmarks/DbfDataReaderBenchmarks.cs b/test/DbfDataReader.Benchmarks/DbfDataReaderBenchmarks.cs index db956d3..8587720 100644 --- a/test/DbfDataReader.Benchmarks/DbfDataReaderBenchmarks.cs +++ b/test/DbfDataReader.Benchmarks/DbfDataReaderBenchmarks.cs @@ -13,6 +13,9 @@ public class DbfDataReaderBenchmarks { private const string FixturePath = "./fixtures/tl_2019_01_place.dbf"; + [GlobalSetup] + public void Setup() => BenchmarkVersion.Print(); + [Benchmark] public void Sylvan() { diff --git a/test/DbfDataReader.Benchmarks/IndexQueryBenchmarks.cs b/test/DbfDataReader.Benchmarks/IndexQueryBenchmarks.cs index 4cff3e1..e111f58 100644 --- a/test/DbfDataReader.Benchmarks/IndexQueryBenchmarks.cs +++ b/test/DbfDataReader.Benchmarks/IndexQueryBenchmarks.cs @@ -20,6 +20,7 @@ public class IndexQueryBenchmarks [GlobalSetup] public void Setup() { + BenchmarkVersion.Print(); _directory = Path.Combine(Path.GetTempPath(), "DbfDataReader.Benchmarks", $"index-{RowCount}"); BenchmarkTableGenerator.Generate(_directory, RowCount); diff --git a/test/DbfDataReader.Benchmarks/README.md b/test/DbfDataReader.Benchmarks/README.md index de09ffb..f74928e 100644 --- a/test/DbfDataReader.Benchmarks/README.md +++ b/test/DbfDataReader.Benchmarks/README.md @@ -3,6 +3,13 @@ Benchmarks run against a **published DbfDataReader package** (default: the version in the csproj). Results are recorded in [`../../benchmarks.md`](../../benchmarks.md). +To benchmark a different version, set the **`DbfDataReaderVersion` environment +variable** — do *not* use `-p:DbfDataReaderVersion`. BenchmarkDotNet rebuilds this +project inside a generated child project which does not inherit `-p` MSBuild +properties, so the `-p` form silently benchmarks the csproj default; the environment +variable reaches every build. Each benchmark prints the version it actually loaded +(`// DbfDataReader under benchmark: ...`) — check it in the log. + ## Cross-library comparison (Sylvan / NDbf / DbfDataReader) Reads every field of every record of `fixtures/tl_2019_01_place.dbf`, in the style of @@ -10,10 +17,12 @@ Reads every field of every record of `fixtures/tl_2019_01_place.dbf`, in the sty Run it against any released version to compare releases: ``` -dotnet run -c Release --project test/DbfDataReader.Benchmarks -p:DbfDataReaderVersion=1.1.0 -- --filter "*DbfDataReaderBenchmarks*" --job short -dotnet run -c Release --project test/DbfDataReader.Benchmarks -p:DbfDataReaderVersion=2.1.0 -- --filter "*DbfDataReaderBenchmarks*" --job short +DbfDataReaderVersion=1.1.0 dotnet run -c Release --project test/DbfDataReader.Benchmarks -- --filter "*DbfDataReaderBenchmarks*" --job short +DbfDataReaderVersion=2.1.0 dotnet run -c Release --project test/DbfDataReader.Benchmarks -- --filter "*DbfDataReaderBenchmarks*" --job short ``` +(On Windows PowerShell: `$env:DbfDataReaderVersion = "2.1.0"` before the command.) + ## Index benchmarks (2.x only) Generates a 50,000-row table with a compound index and pairs the same SQL with diff --git a/test/DbfDataReader.Tests/QueryColumnSubsetTests.cs b/test/DbfDataReader.Tests/QueryColumnSubsetTests.cs new file mode 100644 index 0000000..d8306b4 --- /dev/null +++ b/test/DbfDataReader.Tests/QueryColumnSubsetTests.cs @@ -0,0 +1,506 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Shouldly; +using Xunit; + +namespace DbfDataReader.Tests; + +// Column-subset parsing (issue #296): queries parse only the columns they reference, +// in two phases (filter columns for every candidate row, the rest only for matching +// rows). These tests pin the guard against exposing stale values and prove, against +// full-parse oracles, that every query path returns identical results. +public class QueryColumnSubsetTests +{ + private const string FixtureFolder = "../../../../fixtures"; + private const string Dbase03Path = FixtureFolder + "/dbase_03.dbf"; + + private const int PointIdOrdinal = 0; // Point_ID C(12) + private const int TypeOrdinal = 1; // Type C(20) + private const int DateVisitOrdinal = 8; // Date_Visit D + private const int MaxPdopOrdinal = 10; // Max_PDOP N + + // --- record-level guards --------------------------------------------------- + + [Fact] + public void Unparsed_ordinals_throw_instead_of_exposing_stale_values() + { + using var table = new DbfTable(Dbase03Path); + var record = new DbfRecord(table); + record.EnableSubsetParsing(); + + table.ReadRaw(record).ShouldBeTrue(); + record.TryParseValues(new[] { PointIdOrdinal, MaxPdopOrdinal }).ShouldBeTrue(); + + record.GetValue(PointIdOrdinal).ShouldNotBeNull(); + record.GetValue(MaxPdopOrdinal).ShouldNotBeNull(); + + Should.Throw(() => record.GetValue(TypeOrdinal)); + Should.Throw(() => record.IsNull(TypeOrdinal)); + Should.Throw(() => record.GetStringValue(TypeOrdinal)); + Should.Throw(() => record.GetValue(TypeOrdinal)); + } + + [Fact] + public void Parsing_is_additive_within_a_row() + { + using var table = new DbfTable(Dbase03Path); + var record = new DbfRecord(table); + record.EnableSubsetParsing(); + + table.ReadRaw(record).ShouldBeTrue(); + + record.TryParseValues(new[] { PointIdOrdinal }).ShouldBeTrue(); + Should.Throw(() => record.GetValue(TypeOrdinal)); + + record.TryParseValues(new[] { TypeOrdinal }).ShouldBeTrue(); + record.GetValue(PointIdOrdinal).ShouldNotBeNull(); + record.GetValue(TypeOrdinal).ShouldNotBeNull(); + } + + [Fact] + public void Values_parsed_for_a_previous_row_are_guarded_after_advancing() + { + using var table = new DbfTable(Dbase03Path); + var record = new DbfRecord(table); + record.EnableSubsetParsing(); + + table.ReadRaw(record).ShouldBeTrue(); + record.TryParseValues(new[] { PointIdOrdinal }).ShouldBeTrue(); + var firstRowValue = record.GetValue(PointIdOrdinal); + + table.ReadRaw(record).ShouldBeTrue(); + Should.Throw(() => record.GetValue(PointIdOrdinal)); + + record.TryParseValues(new[] { PointIdOrdinal }).ShouldBeTrue(); + record.GetValue(PointIdOrdinal).ShouldNotBe(firstRowValue); + } + + [Fact] + public void A_full_read_marks_every_column_parsed() + { + using var table = new DbfTable(Dbase03Path); + var record = new DbfRecord(table); + record.EnableSubsetParsing(); + + table.Read(record).ShouldBeTrue(); + + for (var ordinal = 0; ordinal < table.Columns.Count; ordinal++) + { + record.GetValue(ordinal); // must not throw + } + } + + [Fact] + public void Subset_parsing_requires_enabling_first() + { + using var table = new DbfTable(Dbase03Path); + var record = new DbfRecord(table); + + table.ReadRaw(record).ShouldBeTrue(); + + Should.Throw(() => record.TryParseValues(new[] { PointIdOrdinal })); + } + + // --- SQL paths against a full-parse oracle ---------------------------------- + + private sealed record OracleRow(object[] Values, bool IsDeleted); + + private static List ReadOracle(string path) + { + var rows = new List(); + + using var table = new DbfTable(path); + var record = new DbfRecord(table); + while (table.Read(record)) + { + var values = new object[table.Columns.Count]; + for (var ordinal = 0; ordinal < values.Length; ordinal++) + { + values[ordinal] = record.GetValue(ordinal); + } + + rows.Add(new OracleRow(values, record.IsDeleted)); + } + + return rows; + } + + // the median Max_PDOP guarantees the filter both matches and rejects rows, + // so both parse phases are exercised + private static decimal SelectiveThreshold(List oracle) + { + var values = oracle.Select(r => (decimal)r.Values[MaxPdopOrdinal]).OrderBy(v => v).ToList(); + return values[values.Count / 2]; + } + + private static DbfDbConnection OpenConnection() + { + var connection = new DbfDbConnection(); + connection.ConnectionString = $"Folder={FixtureFolder};SkipDeletedRecords=false"; + connection.Open(); + return connection; + } + + private static DbfDbCommand CreateCommand(DbfDbConnection connection, string commandText, + params (string Name, object Value)[] parameters) + { + var command = (DbfDbCommand)connection.CreateCommand(); + command.CommandText = commandText; + foreach (var (name, value) in parameters) + { + command.Parameters.AddWithValue(name, value); + } + + return command; + } + + [Fact] + public void Projection_and_filter_on_disjoint_columns_match_the_oracle() + { + var oracle = ReadOracle(Dbase03Path); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] > threshold) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + expected.ShouldNotBeEmpty(); + expected.Count.ShouldBeLessThan(oracle.Count); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select Point_ID from dbase_03 where Max_PDOP > @pdop", ("pdop", threshold)); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + [Fact] + public void Select_star_with_filter_parses_every_projected_column() + { + var oracle = ReadOracle(Dbase03Path); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] > threshold) + .Select(r => r.Values) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select * from dbase_03 where Max_PDOP > @pdop", ("pdop", threshold)); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var values = new object[reader.FieldCount]; + reader.GetValues(values); + actual.Add(values); + } + + actual.Count.ShouldBe(expected.Count); + for (var row = 0; row < actual.Count; row++) + { + actual[row].ShouldBe(expected[row]); + } + } + + [Fact] + public void Ordering_by_a_column_outside_the_projection_matches_the_oracle() + { + var oracle = ReadOracle(Dbase03Path); + var expected = oracle + .OrderByDescending(r => (decimal)r.Values[MaxPdopOrdinal]) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, "select Point_ID from dbase_03 order by Max_PDOP desc"); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + [Fact] + public void Filter_ordering_and_limit_combine_with_subset_parsing() + { + var oracle = ReadOracle(Dbase03Path); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] >= threshold) + .OrderBy(r => (DateTime?)r.Values[DateVisitOrdinal]) + .Select(r => (string)r.Values[PointIdOrdinal]) + .Take(3) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select Point_ID from dbase_03 where Max_PDOP >= @pdop order by Date_Visit limit 3", + ("pdop", threshold)); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + [Fact] + public void Count_with_filter_parses_only_the_filter_columns() + { + var oracle = ReadOracle(Dbase03Path); + var threshold = SelectiveThreshold(oracle); + var expected = oracle.Count(r => (decimal)r.Values[MaxPdopOrdinal] > threshold); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select count(*) from dbase_03 where Max_PDOP > @pdop", ("pdop", threshold)); + + command.ExecuteScalar().ShouldBe(expected); + } + + [Fact] + public async Task Async_reads_return_the_same_rows() + { + var oracle = ReadOracle(Dbase03Path); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] > threshold) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select Point_ID from dbase_03 where Max_PDOP > @pdop", ("pdop", threshold)); + + var actual = new List(); + using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + [Fact] + public async Task Async_sorted_reads_return_the_same_rows() + { + var oracle = ReadOracle(Dbase03Path); + var expected = oracle + .OrderByDescending(r => (decimal)r.Values[MaxPdopOrdinal]) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, "select Point_ID from dbase_03 order by Max_PDOP desc"); + + var actual = new List(); + using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + // --- memo tables: unreferenced memo columns are never parsed ---------------- + + [Fact] + public void Queries_on_memo_tables_skip_the_memo_column() + { + var oracle = ReadOracle(FixtureFolder + "/dbase_8b.dbf"); + var expected = oracle + .Where(r => r.Values[1] is decimal value && value > 0m) // NUMERICAL + .Select(r => (string)r.Values[0]) // CHARACTER + .ToList(); + expected.ShouldNotBeEmpty(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select CHARACTER from dbase_8b where NUMERICAL > 0"); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + [Fact] + public void Select_star_on_memo_tables_still_returns_memo_content() + { + var oracle = ReadOracle(FixtureFolder + "/dbase_8b.dbf"); + var expected = oracle + .Where(r => r.Values[1] is decimal value && value > 0m) + .Select(r => r.Values) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, "select * from dbase_8b where NUMERICAL > 0"); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var values = new object[reader.FieldCount]; + reader.GetValues(values); + actual.Add(values); + } + + actual.Count.ShouldBe(expected.Count); + for (var row = 0; row < actual.Count; row++) + { + actual[row].ShouldBe(expected[row]); + } + } + + // --- typed query builder ----------------------------------------------------- + + private sealed class PdopPoint + { + public string Point_ID { get; set; } + + public decimal? Max_PDOP { get; set; } + } + + [Fact] + public void Typed_queries_parse_only_the_mapped_columns() + { + var oracle = ReadOracle(Dbase03Path).Where(r => !r.IsDeleted).ToList(); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] > threshold) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + expected.ShouldNotBeEmpty(); + + using var table = new DbfTable(Dbase03Path); + var actual = table.Query() + .Where(p => p.Max_PDOP > threshold) + .ToList(); + + actual.Select(p => p.Point_ID).ShouldBe(expected); + actual.All(p => p.Max_PDOP > threshold).ShouldBeTrue(); + } + + [Fact] + public void Typed_queries_sort_and_limit_with_subset_parsing() + { + var oracle = ReadOracle(Dbase03Path).Where(r => !r.IsDeleted).ToList(); + var expected = oracle + .OrderByDescending(r => (decimal)r.Values[MaxPdopOrdinal]) + .Select(r => (string)r.Values[PointIdOrdinal]) + .Take(5) + .ToList(); + + using var table = new DbfTable(Dbase03Path); + var actual = table.Query() + .OrderByDescending(p => p.Max_PDOP) + .Take(5) + .ToList(); + + actual.Select(p => p.Point_ID).ShouldBe(expected); + } + + [Fact] + public async Task Typed_async_queries_return_the_same_rows() + { + var oracle = ReadOracle(Dbase03Path).Where(r => !r.IsDeleted).ToList(); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] > threshold) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + + using var table = new DbfTable(Dbase03Path); + var actual = await table.Query() + .Where(p => p.Max_PDOP > threshold) + .ToListAsync(); + + actual.Select(p => p.Point_ID).ShouldBe(expected); + } + + [Fact] + public void Typed_count_parses_only_the_filter_columns() + { + var oracle = ReadOracle(Dbase03Path).Where(r => !r.IsDeleted).ToList(); + var threshold = SelectiveThreshold(oracle); + var expected = oracle.Count(r => (decimal)r.Values[MaxPdopOrdinal] > threshold); + + using var table = new DbfTable(Dbase03Path); + table.Query().Where(p => p.Max_PDOP > threshold).Count().ShouldBe(expected); + } +} + +// the foxprodb CDX fixtures are shared with the other index test suites, which +// serialize access through this collection +[Collection("foxprodb")] +public class QueryColumnSubsetIndexTests +{ + private const string FolderPath = "../../../../fixtures/foxprodb"; + + private static DbfDbConnection OpenConnection(bool useIndexes) + { + var connection = new DbfDbConnection(); + connection.ConnectionString = $"Folder={FolderPath};SkipDeletedRecords=false;UseIndexes={useIndexes}"; + connection.Open(); + return connection; + } + + private static List QueryRows(bool useIndexes, string commandText) + { + using var connection = OpenConnection(useIndexes); + var command = (DbfDbCommand)connection.CreateCommand(); + command.CommandText = commandText; + + var rows = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var values = new object[reader.FieldCount]; + reader.GetValues(values); + rows.Add(string.Join("|", values)); + } + + return rows; + } + + [Theory] + [InlineData("select KEY_NAME from setup where KEY_NAME = 'CONTACTS'")] + [InlineData("select * from setup where KEY_NAME = 'CONTACTS'")] + [InlineData("select * from setup where KEY_NAME between 'CALLS' and 'CONTACTS'")] + public void Index_and_scan_paths_return_identical_rows_with_subset_parsing(string commandText) + { + using var connection = OpenConnection(useIndexes: true); + var command = (DbfDbCommand)connection.CreateCommand(); + command.CommandText = commandText; + command.ExplainPlan().ShouldContain("index"); + + var indexed = QueryRows(useIndexes: true, commandText); + var scanned = QueryRows(useIndexes: false, commandText); + + indexed.ShouldBe(scanned); + indexed.ShouldNotBeEmpty(); + } +} From e36fc13380b96bb95904ab2df1f58bde97ed0350 Mon Sep 17 00:00:00 2001 From: Chris Richards Date: Mon, 6 Jul 2026 18:30:39 +0100 Subject: [PATCH 2/3] Address SonarCloud findings Row evaluation collapses to a tri-state EvaluateRow helper shared by the four query loops, the count executor splits by row source, and QueryPlan's parse sets are properties rather than extra constructor parameters. Co-Authored-By: Claude Fable 5 --- src/DbfDataReader/Query/CountExecutor.cs | 36 ++++++++++----- src/DbfDataReader/Query/DbfQuery.cs | 59 ++++++++++++++++-------- 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/src/DbfDataReader/Query/CountExecutor.cs b/src/DbfDataReader/Query/CountExecutor.cs index c13816b..8568b4b 100644 --- a/src/DbfDataReader/Query/CountExecutor.cs +++ b/src/DbfDataReader/Query/CountExecutor.cs @@ -85,26 +85,40 @@ private static int CountByStatusChecks(DbfTable table, DbfRecord record, IReadOn // rows are counted parsing only the columns the WHERE clause references private static int CountByReadingRows(DbfDataReader reader, SqlExpressionEvaluator evaluator, QueryAccessPlan plan, int[] filterOrdinals) + { + return plan.RecordIndexes == null + ? CountBySequentialScan(reader, evaluator, filterOrdinals) + : CountByIndexResult(reader, evaluator, plan.RecordIndexes, filterOrdinals); + } + + private static int CountBySequentialScan(DbfDataReader reader, SqlExpressionEvaluator evaluator, + int[] filterOrdinals) { Func accessor = reader.GetValue; - var table = reader.DbfTable; var record = reader.DbfRecord; var count = 0; - if (plan.RecordIndexes == null) + // reader.ReadRaw applies the skip-deleted option itself + while (reader.ReadRaw()) { - // reader.ReadRaw applies the skip-deleted option itself - while (reader.ReadRaw()) - { - if (!record.TryParseValues(filterOrdinals)) break; - if (evaluator.Matches(accessor)) count++; - } - - return count; + if (!record.TryParseValues(filterOrdinals)) break; + if (evaluator.Matches(accessor)) count++; } - foreach (var recordIndex in plan.RecordIndexes) + return count; + } + + private static int CountByIndexResult(DbfDataReader reader, SqlExpressionEvaluator evaluator, + IReadOnlyList recordIndexes, int[] filterOrdinals) + { + Func accessor = reader.GetValue; + var table = reader.DbfTable; + var record = reader.DbfRecord; + + var count = 0; + + foreach (var recordIndex in recordIndexes) { table.Seek(recordIndex); if (!table.ReadRaw(record)) continue; diff --git a/src/DbfDataReader/Query/DbfQuery.cs b/src/DbfDataReader/Query/DbfQuery.cs index 63c9e17..c4a62c8 100644 --- a/src/DbfDataReader/Query/DbfQuery.cs +++ b/src/DbfDataReader/Query/DbfQuery.cs @@ -191,9 +191,7 @@ private int CountByReadingRows(QueryPlan plan, DbfRecord record, int limit) var count = 0; while (count < limit && ReadNextRow(plan, record, ref position)) { - if (!_includeDeleted && record.IsDeleted) continue; - if (!record.TryParseValues(plan.FilterOrdinals)) break; - if (plan.Filter != null && !plan.Filter.Matches(accessor)) continue; + if (!MatchesFilter(record, plan, accessor)) continue; count++; } @@ -246,8 +244,9 @@ public async IAsyncEnumerable AsAsyncEnumerable( while (NotLimited(returned) && await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(false)) { - if (!MatchesFilter(record, plan, accessor)) continue; - if (!record.TryParseValues(plan.PostFilterOrdinals)) break; + var decision = EvaluateRow(record, plan, accessor); + if (decision == RowDecision.Stop) break; + if (decision == RowDecision.Skip) continue; yield return Materialize(record, plan); returned++; @@ -259,8 +258,9 @@ await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(f var buffer = new List<(T Item, object[] Keys)>(); while (await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(false)) { - if (!MatchesFilter(record, plan, accessor)) continue; - if (!record.TryParseValues(plan.PostFilterOrdinals)) break; + var decision = EvaluateRow(record, plan, accessor); + if (decision == RowDecision.Stop) break; + if (decision == RowDecision.Skip) continue; buffer.Add((Materialize(record, plan), SnapshotSortKeys(record, plan))); } @@ -308,8 +308,9 @@ private IEnumerable Execute() var returned = 0; while (NotLimited(returned) && ReadNextRow(plan, record, ref position)) { - if (!MatchesFilter(record, plan, accessor)) continue; - if (!record.TryParseValues(plan.PostFilterOrdinals)) break; + var decision = EvaluateRow(record, plan, accessor); + if (decision == RowDecision.Stop) break; + if (decision == RowDecision.Skip) continue; yield return Materialize(record, plan); returned++; @@ -321,8 +322,9 @@ private IEnumerable Execute() var buffer = new List<(T Item, object[] Keys)>(); while (ReadNextRow(plan, record, ref position)) { - if (!MatchesFilter(record, plan, accessor)) continue; - if (!record.TryParseValues(plan.PostFilterOrdinals)) break; + var decision = EvaluateRow(record, plan, accessor); + if (decision == RowDecision.Stop) break; + if (decision == RowDecision.Skip) continue; buffer.Add((Materialize(record, plan), SnapshotSortKeys(record, plan))); } @@ -343,6 +345,25 @@ private bool NotLimited(int returned) return _take == null || returned < _take.Value; } + private enum RowDecision + { + Accept, + Skip, + Stop + } + + // the deleted/filter checks parse only the filter's columns; the remaining + // needed columns parse once the row is accepted. Stop means the record data + // ran past the end of a stream (matching Read's end-of-stream contract). + private RowDecision EvaluateRow(DbfRecord record, QueryPlan plan, Func accessor) + { + if (!MatchesFilter(record, plan, accessor)) return RowDecision.Skip; + + return record.TryParseValues(plan.PostFilterOrdinals) ? RowDecision.Accept : RowDecision.Stop; + } + + // counting needs only the filter columns, so this is also the whole of the + // count path's row evaluation private bool MatchesFilter(DbfRecord record, QueryPlan plan, Func accessor) { if (!_includeDeleted && record.IsDeleted) return false; @@ -491,16 +512,18 @@ int ResolveOrdinal(string propertyName) new object[mappedOrdinals.Count], filter, sortKeys, - accessPlan, - SqlColumnCollector.ToSortedOrdinals(filterOrdinals), - SqlColumnCollector.ToSortedOrdinals(needed)); + accessPlan) + { + FilterOrdinals = SqlColumnCollector.ToSortedOrdinals(filterOrdinals), + PostFilterOrdinals = SqlColumnCollector.ToSortedOrdinals(needed) + }; } private sealed class QueryPlan { public QueryPlan(Func materializer, IReadOnlyList mappedOrdinals, object[] rowBuffer, SqlExpressionEvaluator filter, IReadOnlyList<(int Ordinal, bool Descending)> sortKeys, - QueryAccessPlan accessPlan, int[] filterOrdinals, int[] postFilterOrdinals) + QueryAccessPlan accessPlan) { Materializer = materializer; MappedOrdinals = mappedOrdinals; @@ -508,8 +531,6 @@ public QueryPlan(Func materializer, IReadOnlyList mappedOrdina Filter = filter; SortKeys = sortKeys; AccessPlan = accessPlan; - FilterOrdinals = filterOrdinals; - PostFilterOrdinals = postFilterOrdinals; } public Func Materializer { get; } @@ -524,9 +545,9 @@ public QueryPlan(Func materializer, IReadOnlyList mappedOrdina public QueryAccessPlan AccessPlan { get; } - public int[] FilterOrdinals { get; } + public int[] FilterOrdinals { get; set; } - public int[] PostFilterOrdinals { get; } + public int[] PostFilterOrdinals { get; set; } } } } From 47c963ff78b179944636ae2e5afdf203b1d5cde7 Mon Sep 17 00:00:00 2001 From: Chris Richards Date: Mon, 6 Jul 2026 18:33:22 +0100 Subject: [PATCH 3/3] Extract sort-buffer building from the query loops Co-Authored-By: Claude Fable 5 --- src/DbfDataReader/Query/DbfQuery.cs | 33 +++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/DbfDataReader/Query/DbfQuery.cs b/src/DbfDataReader/Query/DbfQuery.cs index c4a62c8..9f3c7f0 100644 --- a/src/DbfDataReader/Query/DbfQuery.cs +++ b/src/DbfDataReader/Query/DbfQuery.cs @@ -255,6 +255,17 @@ await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(f yield break; } + var buffer = await BuildSortBufferAsync(plan, record, accessor, cursor, cancellationToken) + .ConfigureAwait(false); + foreach (var item in SortAndLimit(buffer, plan)) + { + yield return item; + } + } + + private async Task> BuildSortBufferAsync(QueryPlan plan, DbfRecord record, + Func accessor, RowCursor cursor, CancellationToken cancellationToken) + { var buffer = new List<(T Item, object[] Keys)>(); while (await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(false)) { @@ -265,10 +276,7 @@ await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(f buffer.Add((Materialize(record, plan), SnapshotSortKeys(record, plan))); } - foreach (var item in SortAndLimit(buffer, plan)) - { - yield return item; - } + return buffer; } private sealed class RowCursor @@ -302,9 +310,9 @@ private IEnumerable Execute() _table.Seek(0); - var position = 0; if (!SortRequired(plan)) { + var position = 0; var returned = 0; while (NotLimited(returned) && ReadNextRow(plan, record, ref position)) { @@ -319,7 +327,17 @@ private IEnumerable Execute() yield break; } + foreach (var item in SortAndLimit(BuildSortBuffer(plan, record, accessor), plan)) + { + yield return item; + } + } + + private List<(T Item, object[] Keys)> BuildSortBuffer(QueryPlan plan, DbfRecord record, + Func accessor) + { var buffer = new List<(T Item, object[] Keys)>(); + var position = 0; while (ReadNextRow(plan, record, ref position)) { var decision = EvaluateRow(record, plan, accessor); @@ -329,10 +347,7 @@ private IEnumerable Execute() buffer.Add((Materialize(record, plan), SnapshotSortKeys(record, plan))); } - foreach (var item in SortAndLimit(buffer, plan)) - { - yield return item; - } + return buffer; } private static bool SortRequired(QueryPlan plan)