diff --git a/src/DbfDataReader/DbfDataReader.cs b/src/DbfDataReader/DbfDataReader.cs index eeee8cb..06d9d81 100644 --- a/src/DbfDataReader/DbfDataReader.cs +++ b/src/DbfDataReader/DbfDataReader.cs @@ -183,6 +183,40 @@ public override async Task ReadAsync(CancellationToken cancellationToken) return result; } + // advances like Read (honoring SkipDeletedRecords) without parsing any column + // values; callers parse the subset they need via DbfRecord.TryParseValues + internal bool ReadRaw() + { + bool result; + bool skip; + do + { + result = DbfTable.ReadRaw(DbfRecord); + if (!result) + break; + + skip = _options.SkipDeletedRecords && DbfRecord.IsDeleted; + } while (skip); + + return result; + } + + internal async ValueTask ReadRawAsync(CancellationToken cancellationToken) + { + bool result; + bool skip; + do + { + result = await DbfTable.ReadRawAsync(DbfRecord, cancellationToken).ConfigureAwait(false); + if (!result) + break; + + skip = _options.SkipDeletedRecords && DbfRecord.IsDeleted; + } while (skip); + + return result; + } + public void Seek(int recordIndex) { DbfTable.Seek(recordIndex); diff --git a/src/DbfDataReader/DbfRecord.cs b/src/DbfDataReader/DbfRecord.cs index 96c98cf..49ad12b 100644 --- a/src/DbfDataReader/DbfRecord.cs +++ b/src/DbfDataReader/DbfRecord.cs @@ -19,6 +19,12 @@ public class DbfRecord private readonly long _dataOffset; private readonly byte[] _buffer; + // column-subset parsing (issue #296): when enabled, an ordinal's value is only + // readable after it has been parsed for the current row; the stamps guard + // against exposing the previous row's content through the reused value objects + private int[] _parsedVersions; + private int _rowVersion; + public DbfRecord(DbfTable dbfTable) { _encoding = dbfTable.CurrentEncoding; @@ -194,6 +200,8 @@ internal async ValueTask ReadRawAsync(Stream stream, CancellationToken can private bool ReadStatus(long position) { + _rowVersion++; + var status = _buffer[0]; if (status == EndOfFile) return false; @@ -212,21 +220,79 @@ private void ParseValues() var slice = span.Slice(dbfValue.Start, dbfValue.Length); dbfValue.Read(slice); } + + MarkAllParsed(); + } + + // restricts value access to ordinals parsed for the current row; used by the + // query engine, which parses only the columns a query references + internal void EnableSubsetParsing() + { + _parsedVersions ??= new int[Values.Count]; + } + + // parses only the given ordinals for the current row; may be called again with + // further ordinals once the row is known to be needed. Returns false when the + // record data extends past the end of a stream (matching Read's contract). + internal bool TryParseValues(int[] ordinals) + { + if (_parsedVersions == null) + throw new InvalidOperationException( + "EnableSubsetParsing must be called before parsing column subsets."); + + try + { + var span = new ReadOnlySpan(_buffer); + foreach (var ordinal in ordinals) + { + var dbfValue = Values[ordinal]; + dbfValue.Read(span.Slice(dbfValue.Start, dbfValue.Length)); + _parsedVersions[ordinal] = _rowVersion; + } + + return true; + } + catch (EndOfStreamException) + { + return false; + } + } + + private void MarkAllParsed() + { + if (_parsedVersions == null) return; + + for (var ordinal = 0; ordinal < _parsedVersions.Length; ordinal++) + { + _parsedVersions[ordinal] = _rowVersion; + } + } + + private void EnsureParsed(int ordinal) + { + if (_parsedVersions != null && _parsedVersions[ordinal] != _rowVersion) + { + throw new InvalidOperationException( + $"The value at ordinal {ordinal} was not parsed for the current row; the query's column subset does not include it."); + } } public object GetValue(int ordinal) { + EnsureParsed(ordinal); var dbfValue = Values[ordinal]; return dbfValue.GetValue(); } public bool IsNull(int ordinal) { + EnsureParsed(ordinal); return Values[ordinal].IsNull; } public T GetValue(int ordinal) { + EnsureParsed(ordinal); var dbfValue = Values[ordinal]; if (dbfValue is DbfValue typedValue) { @@ -242,6 +308,7 @@ public T GetValue(int ordinal) // avoids boxing the value on the way out internal T GetStructValue(int ordinal) where T : struct { + EnsureParsed(ordinal); if (Values[ordinal] is DbfValue typedValue) { var value = typedValue.Value; @@ -275,6 +342,7 @@ private static SqlNullValueException DataIsNull(int ordinal) public string GetStringValue(int ordinal) { + EnsureParsed(ordinal); var dbfValue = Values[ordinal]; try { diff --git a/src/DbfDataReader/DbfTable.cs b/src/DbfDataReader/DbfTable.cs index b7b567a..5e3f200 100644 --- a/src/DbfDataReader/DbfTable.cs +++ b/src/DbfDataReader/DbfTable.cs @@ -207,6 +207,11 @@ internal bool ReadRaw(DbfRecord dbfRecord) return dbfRecord.ReadRaw(Stream); } + internal ValueTask ReadRawAsync(DbfRecord dbfRecord, CancellationToken cancellationToken = default) + { + return dbfRecord.ReadRawAsync(Stream, cancellationToken); + } + public ValueTask ReadAsync(DbfRecord dbfRecord, CancellationToken cancellationToken = default) { return dbfRecord.ReadAsync(Stream, cancellationToken); diff --git a/src/DbfDataReader/Query/CountExecutor.cs b/src/DbfDataReader/Query/CountExecutor.cs index 71d6eac..8568b4b 100644 --- a/src/DbfDataReader/Query/CountExecutor.cs +++ b/src/DbfDataReader/Query/CountExecutor.cs @@ -48,7 +48,10 @@ public static (int Count, string Description) Execute(SelectStatement statement, } var evaluator = new SqlExpressionEvaluator(statement.Where, namedParameters, positionalParameters); - return (CountByReadingRows(reader, evaluator, plan), description); + var filterOrdinals = + SqlColumnCollector.ToSortedOrdinals(SqlColumnCollector.CollectOrdinals(statement.Where)); + record.EnableSubsetParsing(); + return (CountByReadingRows(reader, evaluator, plan, filterOrdinals), description); } private static int CountByStatusScan(DbfTable table, DbfRecord record, bool skipDeletedRecords) @@ -79,31 +82,48 @@ private static int CountByStatusChecks(DbfTable table, DbfRecord record, IReadOn return count; } + // rows are counted parsing only the columns the WHERE clause references private static int CountByReadingRows(DbfDataReader reader, SqlExpressionEvaluator evaluator, - QueryAccessPlan plan) + QueryAccessPlan plan, int[] filterOrdinals) + { + return plan.RecordIndexes == null + ? CountBySequentialScan(reader, evaluator, filterOrdinals) + : CountByIndexResult(reader, evaluator, plan.RecordIndexes, filterOrdinals); + } + + private static int CountBySequentialScan(DbfDataReader reader, SqlExpressionEvaluator evaluator, + int[] filterOrdinals) { Func accessor = reader.GetValue; - var table = reader.DbfTable; var record = reader.DbfRecord; var count = 0; - if (plan.RecordIndexes == null) + // reader.ReadRaw applies the skip-deleted option itself + while (reader.ReadRaw()) { - // reader.Read applies the skip-deleted option itself - while (reader.Read()) - { - if (evaluator.Matches(accessor)) count++; - } - - return count; + if (!record.TryParseValues(filterOrdinals)) break; + if (evaluator.Matches(accessor)) count++; } - foreach (var recordIndex in plan.RecordIndexes) + return count; + } + + private static int CountByIndexResult(DbfDataReader reader, SqlExpressionEvaluator evaluator, + IReadOnlyList recordIndexes, int[] filterOrdinals) + { + Func accessor = reader.GetValue; + var table = reader.DbfTable; + var record = reader.DbfRecord; + + var count = 0; + + foreach (var recordIndex in recordIndexes) { table.Seek(recordIndex); - if (!table.Read(record)) continue; + if (!table.ReadRaw(record)) continue; if (reader.SkipsDeletedRecords && record.IsDeleted) continue; + if (!record.TryParseValues(filterOrdinals)) break; if (!evaluator.Matches(accessor)) continue; count++; diff --git a/src/DbfDataReader/Query/DbfQuery.cs b/src/DbfDataReader/Query/DbfQuery.cs index d3cfbc0..9f3c7f0 100644 --- a/src/DbfDataReader/Query/DbfQuery.cs +++ b/src/DbfDataReader/Query/DbfQuery.cs @@ -130,6 +130,7 @@ public int Count() { var plan = Prepare(); var record = new DbfRecord(_table); + record.EnableSubsetParsing(); _table.Seek(0); @@ -190,7 +191,7 @@ private int CountByReadingRows(QueryPlan plan, DbfRecord record, int limit) var count = 0; while (count < limit && ReadNextRow(plan, record, ref position)) { - if (!Accept(record, plan, accessor)) continue; + if (!MatchesFilter(record, plan, accessor)) continue; count++; } @@ -203,13 +204,13 @@ private int CountByReadingRows(QueryPlan plan, DbfRecord record, int limit) private bool ReadNextRow(QueryPlan plan, DbfRecord record, ref int position) { var recordIndexes = plan.AccessPlan.RecordIndexes; - if (recordIndexes == null) return _table.Read(record); + if (recordIndexes == null) return _table.ReadRaw(record); while (position < recordIndexes.Count) { _table.Seek(recordIndexes[position]); position++; - if (_table.Read(record)) return true; // false: entry beyond the table + if (_table.ReadRaw(record)) return true; // false: entry beyond the table } return false; @@ -231,6 +232,7 @@ public async IAsyncEnumerable AsAsyncEnumerable( { var plan = Prepare(); var record = new DbfRecord(_table); + record.EnableSubsetParsing(); Func accessor = record.GetValue; var cursor = new RowCursor(); @@ -242,7 +244,9 @@ public async IAsyncEnumerable AsAsyncEnumerable( while (NotLimited(returned) && await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(false)) { - if (!Accept(record, plan, accessor)) continue; + var decision = EvaluateRow(record, plan, accessor); + if (decision == RowDecision.Stop) break; + if (decision == RowDecision.Skip) continue; yield return Materialize(record, plan); returned++; @@ -251,18 +255,28 @@ await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(f yield break; } + var buffer = await BuildSortBufferAsync(plan, record, accessor, cursor, cancellationToken) + .ConfigureAwait(false); + foreach (var item in SortAndLimit(buffer, plan)) + { + yield return item; + } + } + + private async Task> BuildSortBufferAsync(QueryPlan plan, DbfRecord record, + Func accessor, RowCursor cursor, CancellationToken cancellationToken) + { var buffer = new List<(T Item, object[] Keys)>(); while (await ReadNextRowAsync(plan, record, cursor, cancellationToken).ConfigureAwait(false)) { - if (!Accept(record, plan, accessor)) continue; + var decision = EvaluateRow(record, plan, accessor); + if (decision == RowDecision.Stop) break; + if (decision == RowDecision.Skip) continue; buffer.Add((Materialize(record, plan), SnapshotSortKeys(record, plan))); } - foreach (var item in SortAndLimit(buffer, plan)) - { - yield return item; - } + return buffer; } private sealed class RowCursor @@ -275,13 +289,13 @@ private async Task ReadNextRowAsync(QueryPlan plan, DbfRecord record, RowC { var recordIndexes = plan.AccessPlan.RecordIndexes; if (recordIndexes == null) - return await _table.ReadAsync(record, cancellationToken).ConfigureAwait(false); + return await _table.ReadRawAsync(record, cancellationToken).ConfigureAwait(false); while (cursor.Position < recordIndexes.Count) { _table.Seek(recordIndexes[cursor.Position]); cursor.Position++; - if (await _table.ReadAsync(record, cancellationToken).ConfigureAwait(false)) return true; + if (await _table.ReadRawAsync(record, cancellationToken).ConfigureAwait(false)) return true; } return false; @@ -291,17 +305,20 @@ private IEnumerable Execute() { var plan = Prepare(); var record = new DbfRecord(_table); + record.EnableSubsetParsing(); Func accessor = record.GetValue; _table.Seek(0); - var position = 0; if (!SortRequired(plan)) { + var position = 0; var returned = 0; while (NotLimited(returned) && ReadNextRow(plan, record, ref position)) { - if (!Accept(record, plan, accessor)) continue; + var decision = EvaluateRow(record, plan, accessor); + if (decision == RowDecision.Stop) break; + if (decision == RowDecision.Skip) continue; yield return Materialize(record, plan); returned++; @@ -310,18 +327,27 @@ private IEnumerable Execute() yield break; } + foreach (var item in SortAndLimit(BuildSortBuffer(plan, record, accessor), plan)) + { + yield return item; + } + } + + private List<(T Item, object[] Keys)> BuildSortBuffer(QueryPlan plan, DbfRecord record, + Func accessor) + { var buffer = new List<(T Item, object[] Keys)>(); + var position = 0; while (ReadNextRow(plan, record, ref position)) { - if (!Accept(record, plan, accessor)) continue; + var decision = EvaluateRow(record, plan, accessor); + if (decision == RowDecision.Stop) break; + if (decision == RowDecision.Skip) continue; buffer.Add((Materialize(record, plan), SnapshotSortKeys(record, plan))); } - foreach (var item in SortAndLimit(buffer, plan)) - { - yield return item; - } + return buffer; } private static bool SortRequired(QueryPlan plan) @@ -334,9 +360,29 @@ private bool NotLimited(int returned) return _take == null || returned < _take.Value; } - private bool Accept(DbfRecord record, QueryPlan plan, Func accessor) + private enum RowDecision + { + Accept, + Skip, + Stop + } + + // the deleted/filter checks parse only the filter's columns; the remaining + // needed columns parse once the row is accepted. Stop means the record data + // ran past the end of a stream (matching Read's end-of-stream contract). + private RowDecision EvaluateRow(DbfRecord record, QueryPlan plan, Func accessor) + { + if (!MatchesFilter(record, plan, accessor)) return RowDecision.Skip; + + return record.TryParseValues(plan.PostFilterOrdinals) ? RowDecision.Accept : RowDecision.Stop; + } + + // counting needs only the filter columns, so this is also the whole of the + // count path's row evaluation + private bool MatchesFilter(DbfRecord record, QueryPlan plan, Func accessor) { if (!_includeDeleted && record.IsDeleted) return false; + if (!record.TryParseValues(plan.FilterOrdinals)) return false; return plan.Filter == null || plan.Filter.Matches(accessor); } @@ -457,13 +503,35 @@ int ResolveOrdinal(string propertyName) var accessPlan = QueryPlanner.CreatePlan(combinedWhere, sortKeys, _table, _useIndexes, null, null); + // column-subset parsing (issue #296): the filter's columns are parsed for + // every candidate row; the remaining mapped (and, when sorting, sort key) + // columns only for rows that match + var filterOrdinals = combinedWhere == null + ? new HashSet() + : SqlColumnCollector.CollectOrdinals(combinedWhere); + + var needed = new HashSet(mappedOrdinals); + if (sortKeys.Count > 0 && !accessPlan.SortSatisfied) + { + foreach (var (ordinal, _) in sortKeys) + { + needed.Add(ordinal); + } + } + + needed.ExceptWith(filterOrdinals); + return new QueryPlan( RowMaterializer.Create(mappedNames), mappedOrdinals, new object[mappedOrdinals.Count], filter, sortKeys, - accessPlan); + accessPlan) + { + FilterOrdinals = SqlColumnCollector.ToSortedOrdinals(filterOrdinals), + PostFilterOrdinals = SqlColumnCollector.ToSortedOrdinals(needed) + }; } private sealed class QueryPlan @@ -491,6 +559,10 @@ public QueryPlan(Func materializer, IReadOnlyList mappedOrdina public IReadOnlyList<(int Ordinal, bool Descending)> SortKeys { get; } public QueryAccessPlan AccessPlan { get; } + + public int[] FilterOrdinals { get; set; } + + public int[] PostFilterOrdinals { get; set; } } } } diff --git a/src/DbfDataReader/Query/DbfQueryDataReader.cs b/src/DbfDataReader/Query/DbfQueryDataReader.cs index 054ee59..6b0d30a 100644 --- a/src/DbfDataReader/Query/DbfQueryDataReader.cs +++ b/src/DbfDataReader/Query/DbfQueryDataReader.cs @@ -25,6 +25,9 @@ internal sealed class DbfQueryDataReader : DbDataReader, IDbColumnSchemaGenerato private readonly QueryAccessPlan _plan; private readonly bool _skipDeletedRecords; private readonly int _limit; // -1 when unlimited + private readonly int[] _filterParseOrdinals; // parsed before the filter runs + private readonly int[] _postFilterParseOrdinals; // parsed only for matching rows + private readonly int[] _snapshotOrdinals; // ordinals captured into sort-buffer rows private int _rowsReturned; private int _planPosition; // cursor into the plan's record indexes @@ -70,6 +73,30 @@ public DbfQueryDataReader(DbfDataReader reader, SelectStatement statement, SqlEx _ordinals = ordinals; _names = names; _limit = statement.Top ?? -1; + + // column-subset parsing (issue #296): rows are parsed in two phases - the + // columns the filter references first, then, only once a row matches, the + // remaining projected (and, when sorting, ORDER BY) columns + var filterOrdinals = statement.Where == null + ? new HashSet() + : SqlColumnCollector.CollectOrdinals(statement.Where); + + var needed = new HashSet(ordinals); + if (SortRequired) + { + foreach (var item in _orderBy) + { + needed.Add(item.Ordinal); + } + } + + _snapshotOrdinals = SqlColumnCollector.ToSortedOrdinals(needed); + + needed.ExceptWith(filterOrdinals); + _filterParseOrdinals = SqlColumnCollector.ToSortedOrdinals(filterOrdinals); + _postFilterParseOrdinals = SqlColumnCollector.ToSortedOrdinals(needed); + + reader.DbfRecord.EnableSubsetParsing(); } public override int FieldCount => _names.Count; @@ -104,6 +131,7 @@ public override bool Read() while (ReadNextRow()) { if (_filter != null && !_filter.Matches(_readerValueAccessor)) continue; + if (!_reader.DbfRecord.TryParseValues(_postFilterParseOrdinals)) return false; _rowsReturned++; return true; @@ -125,6 +153,7 @@ public override async Task ReadAsync(CancellationToken cancellationToken) while (await ReadNextRowAsync(cancellationToken).ConfigureAwait(false)) { if (_filter != null && !_filter.Matches(_readerValueAccessor)) continue; + if (!_reader.DbfRecord.TryParseValues(_postFilterParseOrdinals)) return false; _rowsReturned++; return true; @@ -133,12 +162,14 @@ public override async Task ReadAsync(CancellationToken cancellationToken) return false; } - // advances the underlying reader to the next candidate row: sequentially, or by + // advances the underlying reader to the next candidate row - sequentially, or by // seeking the next record index supplied by the access plan (re-checking the - // deleted flag, which sequential reads handle inside the raw reader) + // deleted flag, which sequential reads handle inside the raw reader) - parsing + // only the columns the filter needs private bool ReadNextRow() { - if (_plan?.RecordIndexes == null) return _reader.Read(); + if (_plan?.RecordIndexes == null) + return _reader.ReadRaw() && _reader.DbfRecord.TryParseValues(_filterParseOrdinals); while (_planPosition < _plan.RecordIndexes.Count) { @@ -146,10 +177,10 @@ private bool ReadNextRow() _planPosition++; _reader.Seek(recordIndex); - if (!_reader.DbfTable.Read(_reader.DbfRecord)) continue; // entry beyond the table + if (!_reader.DbfTable.ReadRaw(_reader.DbfRecord)) continue; // entry beyond the table if (_skipDeletedRecords && _reader.DbfRecord.IsDeleted) continue; - return true; + return _reader.DbfRecord.TryParseValues(_filterParseOrdinals); } return false; @@ -158,7 +189,10 @@ private bool ReadNextRow() private async Task ReadNextRowAsync(CancellationToken cancellationToken) { if (_plan?.RecordIndexes == null) - return await _reader.ReadAsync(cancellationToken).ConfigureAwait(false); + { + return await _reader.ReadRawAsync(cancellationToken).ConfigureAwait(false) && + _reader.DbfRecord.TryParseValues(_filterParseOrdinals); + } while (_planPosition < _plan.RecordIndexes.Count) { @@ -166,11 +200,11 @@ private async Task ReadNextRowAsync(CancellationToken cancellationToken) _planPosition++; _reader.Seek(recordIndex); - if (!await _reader.DbfTable.ReadAsync(_reader.DbfRecord, cancellationToken).ConfigureAwait(false)) + if (!await _reader.DbfTable.ReadRawAsync(_reader.DbfRecord, cancellationToken).ConfigureAwait(false)) continue; if (_skipDeletedRecords && _reader.DbfRecord.IsDeleted) continue; - return true; + return _reader.DbfRecord.TryParseValues(_filterParseOrdinals); } return false; @@ -192,6 +226,7 @@ private List BuildSortedRows() while (ReadNextRow()) { if (_filter != null && !_filter.Matches(_readerValueAccessor)) continue; + if (!_reader.DbfRecord.TryParseValues(_postFilterParseOrdinals)) break; rows.Add(SnapshotCurrentRow()); } @@ -206,6 +241,7 @@ private async Task> BuildSortedRowsAsync(CancellationToken cancel while (await ReadNextRowAsync(cancellationToken).ConfigureAwait(false)) { if (_filter != null && !_filter.Matches(_readerValueAccessor)) continue; + if (!_reader.DbfRecord.TryParseValues(_postFilterParseOrdinals)) break; rows.Add(SnapshotCurrentRow()); } @@ -214,10 +250,12 @@ private async Task> BuildSortedRowsAsync(CancellationToken cancel return rows; } + // full-width so underlying ordinals index it directly; only the projected and + // ORDER BY positions are populated (and only those are ever read back) private object[] SnapshotCurrentRow() { var row = new object[_reader.FieldCount]; - for (var ordinal = 0; ordinal < row.Length; ordinal++) + foreach (var ordinal in _snapshotOrdinals) { row[ordinal] = _reader.GetValue(ordinal); } diff --git a/src/DbfDataReader/Query/SqlColumnCollector.cs b/src/DbfDataReader/Query/SqlColumnCollector.cs new file mode 100644 index 0000000..9471a61 --- /dev/null +++ b/src/DbfDataReader/Query/SqlColumnCollector.cs @@ -0,0 +1,65 @@ +using System.Collections.Generic; + +namespace DbfDataReader.Query +{ + // collects the ordinals of every column a bound expression references, so the + // engine can parse only the values a query actually needs (issue #296) + internal static class SqlColumnCollector + { + public static HashSet CollectOrdinals(SqlExpression expression) + { + var ordinals = new HashSet(); + Collect(expression, ordinals); + return ordinals; + } + + public static int[] ToSortedOrdinals(HashSet ordinals) + { + var result = new int[ordinals.Count]; + ordinals.CopyTo(result); + System.Array.Sort(result); + return result; + } + + private static void Collect(SqlExpression expression, HashSet ordinals) + { + switch (expression) + { + case null: + return; + case SqlColumnExpression column: + ordinals.Add(column.Ordinal); + return; + case SqlBinaryExpression binary: + Collect(binary.Left, ordinals); + Collect(binary.Right, ordinals); + return; + case SqlNotExpression not: + Collect(not.Operand, ordinals); + return; + case SqlBetweenExpression between: + Collect(between.Operand, ordinals); + Collect(between.Low, ordinals); + Collect(between.High, ordinals); + return; + case SqlInExpression inExpression: + Collect(inExpression.Operand, ordinals); + foreach (var value in inExpression.Values) + { + Collect(value, ordinals); + } + + return; + case SqlLikeExpression like: + Collect(like.Operand, ordinals); + Collect(like.Pattern, ordinals); + return; + case SqlIsNullExpression isNull: + Collect(isNull.Operand, ordinals); + return; + default: + return; // literals and parameters reference no columns + } + } + } +} diff --git a/test/DbfDataReader.Benchmarks/BenchmarkVersion.cs b/test/DbfDataReader.Benchmarks/BenchmarkVersion.cs new file mode 100644 index 0000000..e09b94c --- /dev/null +++ b/test/DbfDataReader.Benchmarks/BenchmarkVersion.cs @@ -0,0 +1,20 @@ +using System; +using System.Reflection; + +namespace DbfDataReader.Benchmarks +{ + // BenchmarkDotNet rebuilds this project in a generated child project, which does + // NOT inherit -p:DbfDataReaderVersion from the command line (only the environment + // variable form reaches it - see the README). Printing the version the benchmark + // process actually loaded makes a silently wrong package impossible to miss. + internal static class BenchmarkVersion + { + public static void Print() + { + var assembly = typeof(DbfDataReader).Assembly; + var version = assembly.GetCustomAttribute() + ?.InformationalVersion ?? assembly.GetName().Version?.ToString(); + Console.WriteLine($"// DbfDataReader under benchmark: {version}"); + } + } +} diff --git a/test/DbfDataReader.Benchmarks/DbfDataReaderBenchmarks.cs b/test/DbfDataReader.Benchmarks/DbfDataReaderBenchmarks.cs index db956d3..8587720 100644 --- a/test/DbfDataReader.Benchmarks/DbfDataReaderBenchmarks.cs +++ b/test/DbfDataReader.Benchmarks/DbfDataReaderBenchmarks.cs @@ -13,6 +13,9 @@ public class DbfDataReaderBenchmarks { private const string FixturePath = "./fixtures/tl_2019_01_place.dbf"; + [GlobalSetup] + public void Setup() => BenchmarkVersion.Print(); + [Benchmark] public void Sylvan() { diff --git a/test/DbfDataReader.Benchmarks/IndexQueryBenchmarks.cs b/test/DbfDataReader.Benchmarks/IndexQueryBenchmarks.cs index 4cff3e1..e111f58 100644 --- a/test/DbfDataReader.Benchmarks/IndexQueryBenchmarks.cs +++ b/test/DbfDataReader.Benchmarks/IndexQueryBenchmarks.cs @@ -20,6 +20,7 @@ public class IndexQueryBenchmarks [GlobalSetup] public void Setup() { + BenchmarkVersion.Print(); _directory = Path.Combine(Path.GetTempPath(), "DbfDataReader.Benchmarks", $"index-{RowCount}"); BenchmarkTableGenerator.Generate(_directory, RowCount); diff --git a/test/DbfDataReader.Benchmarks/README.md b/test/DbfDataReader.Benchmarks/README.md index de09ffb..f74928e 100644 --- a/test/DbfDataReader.Benchmarks/README.md +++ b/test/DbfDataReader.Benchmarks/README.md @@ -3,6 +3,13 @@ Benchmarks run against a **published DbfDataReader package** (default: the version in the csproj). Results are recorded in [`../../benchmarks.md`](../../benchmarks.md). +To benchmark a different version, set the **`DbfDataReaderVersion` environment +variable** — do *not* use `-p:DbfDataReaderVersion`. BenchmarkDotNet rebuilds this +project inside a generated child project which does not inherit `-p` MSBuild +properties, so the `-p` form silently benchmarks the csproj default; the environment +variable reaches every build. Each benchmark prints the version it actually loaded +(`// DbfDataReader under benchmark: ...`) — check it in the log. + ## Cross-library comparison (Sylvan / NDbf / DbfDataReader) Reads every field of every record of `fixtures/tl_2019_01_place.dbf`, in the style of @@ -10,10 +17,12 @@ Reads every field of every record of `fixtures/tl_2019_01_place.dbf`, in the sty Run it against any released version to compare releases: ``` -dotnet run -c Release --project test/DbfDataReader.Benchmarks -p:DbfDataReaderVersion=1.1.0 -- --filter "*DbfDataReaderBenchmarks*" --job short -dotnet run -c Release --project test/DbfDataReader.Benchmarks -p:DbfDataReaderVersion=2.1.0 -- --filter "*DbfDataReaderBenchmarks*" --job short +DbfDataReaderVersion=1.1.0 dotnet run -c Release --project test/DbfDataReader.Benchmarks -- --filter "*DbfDataReaderBenchmarks*" --job short +DbfDataReaderVersion=2.1.0 dotnet run -c Release --project test/DbfDataReader.Benchmarks -- --filter "*DbfDataReaderBenchmarks*" --job short ``` +(On Windows PowerShell: `$env:DbfDataReaderVersion = "2.1.0"` before the command.) + ## Index benchmarks (2.x only) Generates a 50,000-row table with a compound index and pairs the same SQL with diff --git a/test/DbfDataReader.Tests/QueryColumnSubsetTests.cs b/test/DbfDataReader.Tests/QueryColumnSubsetTests.cs new file mode 100644 index 0000000..d8306b4 --- /dev/null +++ b/test/DbfDataReader.Tests/QueryColumnSubsetTests.cs @@ -0,0 +1,506 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Shouldly; +using Xunit; + +namespace DbfDataReader.Tests; + +// Column-subset parsing (issue #296): queries parse only the columns they reference, +// in two phases (filter columns for every candidate row, the rest only for matching +// rows). These tests pin the guard against exposing stale values and prove, against +// full-parse oracles, that every query path returns identical results. +public class QueryColumnSubsetTests +{ + private const string FixtureFolder = "../../../../fixtures"; + private const string Dbase03Path = FixtureFolder + "/dbase_03.dbf"; + + private const int PointIdOrdinal = 0; // Point_ID C(12) + private const int TypeOrdinal = 1; // Type C(20) + private const int DateVisitOrdinal = 8; // Date_Visit D + private const int MaxPdopOrdinal = 10; // Max_PDOP N + + // --- record-level guards --------------------------------------------------- + + [Fact] + public void Unparsed_ordinals_throw_instead_of_exposing_stale_values() + { + using var table = new DbfTable(Dbase03Path); + var record = new DbfRecord(table); + record.EnableSubsetParsing(); + + table.ReadRaw(record).ShouldBeTrue(); + record.TryParseValues(new[] { PointIdOrdinal, MaxPdopOrdinal }).ShouldBeTrue(); + + record.GetValue(PointIdOrdinal).ShouldNotBeNull(); + record.GetValue(MaxPdopOrdinal).ShouldNotBeNull(); + + Should.Throw(() => record.GetValue(TypeOrdinal)); + Should.Throw(() => record.IsNull(TypeOrdinal)); + Should.Throw(() => record.GetStringValue(TypeOrdinal)); + Should.Throw(() => record.GetValue(TypeOrdinal)); + } + + [Fact] + public void Parsing_is_additive_within_a_row() + { + using var table = new DbfTable(Dbase03Path); + var record = new DbfRecord(table); + record.EnableSubsetParsing(); + + table.ReadRaw(record).ShouldBeTrue(); + + record.TryParseValues(new[] { PointIdOrdinal }).ShouldBeTrue(); + Should.Throw(() => record.GetValue(TypeOrdinal)); + + record.TryParseValues(new[] { TypeOrdinal }).ShouldBeTrue(); + record.GetValue(PointIdOrdinal).ShouldNotBeNull(); + record.GetValue(TypeOrdinal).ShouldNotBeNull(); + } + + [Fact] + public void Values_parsed_for_a_previous_row_are_guarded_after_advancing() + { + using var table = new DbfTable(Dbase03Path); + var record = new DbfRecord(table); + record.EnableSubsetParsing(); + + table.ReadRaw(record).ShouldBeTrue(); + record.TryParseValues(new[] { PointIdOrdinal }).ShouldBeTrue(); + var firstRowValue = record.GetValue(PointIdOrdinal); + + table.ReadRaw(record).ShouldBeTrue(); + Should.Throw(() => record.GetValue(PointIdOrdinal)); + + record.TryParseValues(new[] { PointIdOrdinal }).ShouldBeTrue(); + record.GetValue(PointIdOrdinal).ShouldNotBe(firstRowValue); + } + + [Fact] + public void A_full_read_marks_every_column_parsed() + { + using var table = new DbfTable(Dbase03Path); + var record = new DbfRecord(table); + record.EnableSubsetParsing(); + + table.Read(record).ShouldBeTrue(); + + for (var ordinal = 0; ordinal < table.Columns.Count; ordinal++) + { + record.GetValue(ordinal); // must not throw + } + } + + [Fact] + public void Subset_parsing_requires_enabling_first() + { + using var table = new DbfTable(Dbase03Path); + var record = new DbfRecord(table); + + table.ReadRaw(record).ShouldBeTrue(); + + Should.Throw(() => record.TryParseValues(new[] { PointIdOrdinal })); + } + + // --- SQL paths against a full-parse oracle ---------------------------------- + + private sealed record OracleRow(object[] Values, bool IsDeleted); + + private static List ReadOracle(string path) + { + var rows = new List(); + + using var table = new DbfTable(path); + var record = new DbfRecord(table); + while (table.Read(record)) + { + var values = new object[table.Columns.Count]; + for (var ordinal = 0; ordinal < values.Length; ordinal++) + { + values[ordinal] = record.GetValue(ordinal); + } + + rows.Add(new OracleRow(values, record.IsDeleted)); + } + + return rows; + } + + // the median Max_PDOP guarantees the filter both matches and rejects rows, + // so both parse phases are exercised + private static decimal SelectiveThreshold(List oracle) + { + var values = oracle.Select(r => (decimal)r.Values[MaxPdopOrdinal]).OrderBy(v => v).ToList(); + return values[values.Count / 2]; + } + + private static DbfDbConnection OpenConnection() + { + var connection = new DbfDbConnection(); + connection.ConnectionString = $"Folder={FixtureFolder};SkipDeletedRecords=false"; + connection.Open(); + return connection; + } + + private static DbfDbCommand CreateCommand(DbfDbConnection connection, string commandText, + params (string Name, object Value)[] parameters) + { + var command = (DbfDbCommand)connection.CreateCommand(); + command.CommandText = commandText; + foreach (var (name, value) in parameters) + { + command.Parameters.AddWithValue(name, value); + } + + return command; + } + + [Fact] + public void Projection_and_filter_on_disjoint_columns_match_the_oracle() + { + var oracle = ReadOracle(Dbase03Path); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] > threshold) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + expected.ShouldNotBeEmpty(); + expected.Count.ShouldBeLessThan(oracle.Count); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select Point_ID from dbase_03 where Max_PDOP > @pdop", ("pdop", threshold)); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + [Fact] + public void Select_star_with_filter_parses_every_projected_column() + { + var oracle = ReadOracle(Dbase03Path); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] > threshold) + .Select(r => r.Values) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select * from dbase_03 where Max_PDOP > @pdop", ("pdop", threshold)); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var values = new object[reader.FieldCount]; + reader.GetValues(values); + actual.Add(values); + } + + actual.Count.ShouldBe(expected.Count); + for (var row = 0; row < actual.Count; row++) + { + actual[row].ShouldBe(expected[row]); + } + } + + [Fact] + public void Ordering_by_a_column_outside_the_projection_matches_the_oracle() + { + var oracle = ReadOracle(Dbase03Path); + var expected = oracle + .OrderByDescending(r => (decimal)r.Values[MaxPdopOrdinal]) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, "select Point_ID from dbase_03 order by Max_PDOP desc"); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + [Fact] + public void Filter_ordering_and_limit_combine_with_subset_parsing() + { + var oracle = ReadOracle(Dbase03Path); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] >= threshold) + .OrderBy(r => (DateTime?)r.Values[DateVisitOrdinal]) + .Select(r => (string)r.Values[PointIdOrdinal]) + .Take(3) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select Point_ID from dbase_03 where Max_PDOP >= @pdop order by Date_Visit limit 3", + ("pdop", threshold)); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + [Fact] + public void Count_with_filter_parses_only_the_filter_columns() + { + var oracle = ReadOracle(Dbase03Path); + var threshold = SelectiveThreshold(oracle); + var expected = oracle.Count(r => (decimal)r.Values[MaxPdopOrdinal] > threshold); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select count(*) from dbase_03 where Max_PDOP > @pdop", ("pdop", threshold)); + + command.ExecuteScalar().ShouldBe(expected); + } + + [Fact] + public async Task Async_reads_return_the_same_rows() + { + var oracle = ReadOracle(Dbase03Path); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] > threshold) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select Point_ID from dbase_03 where Max_PDOP > @pdop", ("pdop", threshold)); + + var actual = new List(); + using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + [Fact] + public async Task Async_sorted_reads_return_the_same_rows() + { + var oracle = ReadOracle(Dbase03Path); + var expected = oracle + .OrderByDescending(r => (decimal)r.Values[MaxPdopOrdinal]) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, "select Point_ID from dbase_03 order by Max_PDOP desc"); + + var actual = new List(); + using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + // --- memo tables: unreferenced memo columns are never parsed ---------------- + + [Fact] + public void Queries_on_memo_tables_skip_the_memo_column() + { + var oracle = ReadOracle(FixtureFolder + "/dbase_8b.dbf"); + var expected = oracle + .Where(r => r.Values[1] is decimal value && value > 0m) // NUMERICAL + .Select(r => (string)r.Values[0]) // CHARACTER + .ToList(); + expected.ShouldNotBeEmpty(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, + "select CHARACTER from dbase_8b where NUMERICAL > 0"); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + actual.Add(reader.GetString(0)); + } + + actual.ShouldBe(expected); + } + + [Fact] + public void Select_star_on_memo_tables_still_returns_memo_content() + { + var oracle = ReadOracle(FixtureFolder + "/dbase_8b.dbf"); + var expected = oracle + .Where(r => r.Values[1] is decimal value && value > 0m) + .Select(r => r.Values) + .ToList(); + + using var connection = OpenConnection(); + var command = CreateCommand(connection, "select * from dbase_8b where NUMERICAL > 0"); + + var actual = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var values = new object[reader.FieldCount]; + reader.GetValues(values); + actual.Add(values); + } + + actual.Count.ShouldBe(expected.Count); + for (var row = 0; row < actual.Count; row++) + { + actual[row].ShouldBe(expected[row]); + } + } + + // --- typed query builder ----------------------------------------------------- + + private sealed class PdopPoint + { + public string Point_ID { get; set; } + + public decimal? Max_PDOP { get; set; } + } + + [Fact] + public void Typed_queries_parse_only_the_mapped_columns() + { + var oracle = ReadOracle(Dbase03Path).Where(r => !r.IsDeleted).ToList(); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] > threshold) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + expected.ShouldNotBeEmpty(); + + using var table = new DbfTable(Dbase03Path); + var actual = table.Query() + .Where(p => p.Max_PDOP > threshold) + .ToList(); + + actual.Select(p => p.Point_ID).ShouldBe(expected); + actual.All(p => p.Max_PDOP > threshold).ShouldBeTrue(); + } + + [Fact] + public void Typed_queries_sort_and_limit_with_subset_parsing() + { + var oracle = ReadOracle(Dbase03Path).Where(r => !r.IsDeleted).ToList(); + var expected = oracle + .OrderByDescending(r => (decimal)r.Values[MaxPdopOrdinal]) + .Select(r => (string)r.Values[PointIdOrdinal]) + .Take(5) + .ToList(); + + using var table = new DbfTable(Dbase03Path); + var actual = table.Query() + .OrderByDescending(p => p.Max_PDOP) + .Take(5) + .ToList(); + + actual.Select(p => p.Point_ID).ShouldBe(expected); + } + + [Fact] + public async Task Typed_async_queries_return_the_same_rows() + { + var oracle = ReadOracle(Dbase03Path).Where(r => !r.IsDeleted).ToList(); + var threshold = SelectiveThreshold(oracle); + var expected = oracle + .Where(r => (decimal)r.Values[MaxPdopOrdinal] > threshold) + .Select(r => (string)r.Values[PointIdOrdinal]) + .ToList(); + + using var table = new DbfTable(Dbase03Path); + var actual = await table.Query() + .Where(p => p.Max_PDOP > threshold) + .ToListAsync(); + + actual.Select(p => p.Point_ID).ShouldBe(expected); + } + + [Fact] + public void Typed_count_parses_only_the_filter_columns() + { + var oracle = ReadOracle(Dbase03Path).Where(r => !r.IsDeleted).ToList(); + var threshold = SelectiveThreshold(oracle); + var expected = oracle.Count(r => (decimal)r.Values[MaxPdopOrdinal] > threshold); + + using var table = new DbfTable(Dbase03Path); + table.Query().Where(p => p.Max_PDOP > threshold).Count().ShouldBe(expected); + } +} + +// the foxprodb CDX fixtures are shared with the other index test suites, which +// serialize access through this collection +[Collection("foxprodb")] +public class QueryColumnSubsetIndexTests +{ + private const string FolderPath = "../../../../fixtures/foxprodb"; + + private static DbfDbConnection OpenConnection(bool useIndexes) + { + var connection = new DbfDbConnection(); + connection.ConnectionString = $"Folder={FolderPath};SkipDeletedRecords=false;UseIndexes={useIndexes}"; + connection.Open(); + return connection; + } + + private static List QueryRows(bool useIndexes, string commandText) + { + using var connection = OpenConnection(useIndexes); + var command = (DbfDbCommand)connection.CreateCommand(); + command.CommandText = commandText; + + var rows = new List(); + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var values = new object[reader.FieldCount]; + reader.GetValues(values); + rows.Add(string.Join("|", values)); + } + + return rows; + } + + [Theory] + [InlineData("select KEY_NAME from setup where KEY_NAME = 'CONTACTS'")] + [InlineData("select * from setup where KEY_NAME = 'CONTACTS'")] + [InlineData("select * from setup where KEY_NAME between 'CALLS' and 'CONTACTS'")] + public void Index_and_scan_paths_return_identical_rows_with_subset_parsing(string commandText) + { + using var connection = OpenConnection(useIndexes: true); + var command = (DbfDbCommand)connection.CreateCommand(); + command.CommandText = commandText; + command.ExplainPlan().ShouldContain("index"); + + var indexed = QueryRows(useIndexes: true, commandText); + var scanned = QueryRows(useIndexes: false, commandText); + + indexed.ShouldBe(scanned); + indexed.ShouldNotBeEmpty(); + } +}