Parse only the columns a query needs (#296)

src/DbfDataReader/DbfDataReader.cs

@@ -183,6 +183,40 @@ public override async Task<bool> ReadAsync(CancellationToken cancellationToken)
             return result;
         }
 
+        // advances like Read (honoring SkipDeletedRecords) without parsing any column
+        // values; callers parse the subset they need via DbfRecord.TryParseValues
+        internal bool ReadRaw()
+        {
+            bool result;
+            bool skip;
+            do
+            {
+                result = DbfTable.ReadRaw(DbfRecord);
+                if (!result)
+                    break;
+
+                skip = _options.SkipDeletedRecords && DbfRecord.IsDeleted;
+            } while (skip);
+
+            return result;
+        }
+
+        internal async ValueTask<bool> ReadRawAsync(CancellationToken cancellationToken)
+        {
+            bool result;
+            bool skip;
+            do
+            {
+                result = await DbfTable.ReadRawAsync(DbfRecord, cancellationToken).ConfigureAwait(false);
+                if (!result)
+                    break;
+
+                skip = _options.SkipDeletedRecords && DbfRecord.IsDeleted;
+            } while (skip);
+
+            return result;
+        }
+
         public void Seek(int recordIndex)
         {
             DbfTable.Seek(recordIndex);

src/DbfDataReader/DbfRecord.cs

@@ -19,6 +19,12 @@ public class DbfRecord
         private readonly long _dataOffset;
         private readonly byte[] _buffer;
 
+        // column-subset parsing (issue #296): when enabled, an ordinal's value is only
+        // readable after it has been parsed for the current row; the stamps guard
+        // against exposing the previous row's content through the reused value objects
+        private int[] _parsedVersions;
+        private int _rowVersion;
+
         public DbfRecord(DbfTable dbfTable)
         {
             _encoding = dbfTable.CurrentEncoding;
@@ -194,6 +200,8 @@ internal async ValueTask<bool> ReadRawAsync(Stream stream, CancellationToken can
 
         private bool ReadStatus(long position)
         {
+            _rowVersion++;
+
             var status = _buffer[0];
             if (status == EndOfFile) return false;
 
@@ -212,21 +220,79 @@ private void ParseValues()
                 var slice = span.Slice(dbfValue.Start, dbfValue.Length);
                 dbfValue.Read(slice);
             }
+
+            MarkAllParsed();
+        }
+
+        // restricts value access to ordinals parsed for the current row; used by the
+        // query engine, which parses only the columns a query references
+        internal void EnableSubsetParsing()
+        {
+            _parsedVersions ??= new int[Values.Count];
+        }
+
+        // parses only the given ordinals for the current row; may be called again with
+        // further ordinals once the row is known to be needed. Returns false when the
+        // record data extends past the end of a stream (matching Read's contract).
+        internal bool TryParseValues(int[] ordinals)
+        {
+            if (_parsedVersions == null)
+                throw new InvalidOperationException(
+                    "EnableSubsetParsing must be called before parsing column subsets.");
+
+            try
+            {
+                var span = new ReadOnlySpan<byte>(_buffer);
+                foreach (var ordinal in ordinals)
+                {
+                    var dbfValue = Values[ordinal];
+                    dbfValue.Read(span.Slice(dbfValue.Start, dbfValue.Length));
+                    _parsedVersions[ordinal] = _rowVersion;
+                }
+
+                return true;
+            }
+            catch (EndOfStreamException)
+            {
+                return false;
+            }
+        }
+
+        private void MarkAllParsed()
+        {
+            if (_parsedVersions == null) return;
+
+            for (var ordinal = 0; ordinal < _parsedVersions.Length; ordinal++)
+            {
+                _parsedVersions[ordinal] = _rowVersion;
+            }
+        }
+
+        private void EnsureParsed(int ordinal)
+        {
+            if (_parsedVersions != null && _parsedVersions[ordinal] != _rowVersion)
+            {
+                throw new InvalidOperationException(
+                    $"The value at ordinal {ordinal} was not parsed for the current row; the query's column subset does not include it.");
+            }
         }
 
         public object GetValue(int ordinal)
         {
+            EnsureParsed(ordinal);
             var dbfValue = Values[ordinal];
             return dbfValue.GetValue();
         }
 
         public bool IsNull(int ordinal)
         {
+            EnsureParsed(ordinal);
             return Values[ordinal].IsNull;
         }
 
         public T GetValue<T>(int ordinal)
         {
+            EnsureParsed(ordinal);
             var dbfValue = Values[ordinal];
             if (dbfValue is DbfValue<T> typedValue)
             {
@@ -242,6 +308,7 @@ public T GetValue<T>(int ordinal)
         // avoids boxing the value on the way out
         internal T GetStructValue<T>(int ordinal) where T : struct
         {
+            EnsureParsed(ordinal);
             if (Values[ordinal] is DbfValue<T?> typedValue)
             {
                 var value = typedValue.Value;
@@ -275,6 +342,7 @@ private static SqlNullValueException DataIsNull(int ordinal)
 
         public string GetStringValue(int ordinal)
         {
+            EnsureParsed(ordinal);
             var dbfValue = Values[ordinal];
             try
             {

src/DbfDataReader/DbfTable.cs

@@ -207,6 +207,11 @@ internal bool ReadRaw(DbfRecord dbfRecord)
             return dbfRecord.ReadRaw(Stream);
         }
 
+        internal ValueTask<bool> ReadRawAsync(DbfRecord dbfRecord, CancellationToken cancellationToken = default)
+        {
+            return dbfRecord.ReadRawAsync(Stream, cancellationToken);
+        }
+
         public ValueTask<bool> ReadAsync(DbfRecord dbfRecord, CancellationToken cancellationToken = default)
         {
             return dbfRecord.ReadAsync(Stream, cancellationToken);

src/DbfDataReader/Query/CountExecutor.cs

@@ -48,7 +48,10 @@ public static (int Count, string Description) Execute(SelectStatement statement,
             }
 
             var evaluator = new SqlExpressionEvaluator(statement.Where, namedParameters, positionalParameters);
-            return (CountByReadingRows(reader, evaluator, plan), description);
+            var filterOrdinals =
+                SqlColumnCollector.ToSortedOrdinals(SqlColumnCollector.CollectOrdinals(statement.Where));
+            record.EnableSubsetParsing();
+            return (CountByReadingRows(reader, evaluator, plan, filterOrdinals), description);
         }
 
         private static int CountByStatusScan(DbfTable table, DbfRecord record, bool skipDeletedRecords)
@@ -79,31 +82,48 @@ private static int CountByStatusChecks(DbfTable table, DbfRecord record, IReadOn
             return count;
         }
 
+        // rows are counted parsing only the columns the WHERE clause references
         private static int CountByReadingRows(DbfDataReader reader, SqlExpressionEvaluator evaluator,
-            QueryAccessPlan plan)
+            QueryAccessPlan plan, int[] filterOrdinals)
+        {
+            return plan.RecordIndexes == null
+                ? CountBySequentialScan(reader, evaluator, filterOrdinals)
+                : CountByIndexResult(reader, evaluator, plan.RecordIndexes, filterOrdinals);
+        }
+
+        private static int CountBySequentialScan(DbfDataReader reader, SqlExpressionEvaluator evaluator,
+            int[] filterOrdinals)
         {
             Func<int, object> accessor = reader.GetValue;
-            var table = reader.DbfTable;
             var record = reader.DbfRecord;
 
             var count = 0;
 
-            if (plan.RecordIndexes == null)
+            // reader.ReadRaw applies the skip-deleted option itself
+            while (reader.ReadRaw())
             {
-                // reader.Read applies the skip-deleted option itself
-                while (reader.Read())
-                {
-                    if (evaluator.Matches(accessor)) count++;
-                }
-
-                return count;
+                if (!record.TryParseValues(filterOrdinals)) break;
+                if (evaluator.Matches(accessor)) count++;
             }
 
-            foreach (var recordIndex in plan.RecordIndexes)
+            return count;
+        }
+
+        private static int CountByIndexResult(DbfDataReader reader, SqlExpressionEvaluator evaluator,
+            IReadOnlyList<int> recordIndexes, int[] filterOrdinals)
+        {
+            Func<int, object> accessor = reader.GetValue;
+            var table = reader.DbfTable;
+            var record = reader.DbfRecord;
+
+            var count = 0;
+
+            foreach (var recordIndex in recordIndexes)
             {
                 table.Seek(recordIndex);
-                if (!table.Read(record)) continue;
+                if (!table.ReadRaw(record)) continue;
                 if (reader.SkipsDeletedRecords && record.IsDeleted) continue;
+                if (!record.TryParseValues(filterOrdinals)) break;
                 if (!evaluator.Matches(accessor)) continue;
 
                 count++;