Skip to content

Commit b9507d8

Browse files
jackrossclaude
andcommitted
fix(seed): keep NA-marker NULLs when restoring declared columns
On its float parse path pandas matches `na_values` markers against the parsed number as well as the raw string, so with `na_values=[1]` a value like `1.00` is read as NULL. The `dtype=str` re-read used to restore declared columns only does the string match, so assigning its values back turned such NULLs into `'1.00'` in declared TEXT columns. Restored TEXT and integer columns now keep the null positions of the original read, which matches how the CSV loads without declared columns. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Jack A Ross (gmatter) <jack.ross@gmatter.co>
1 parent 4459cc1 commit b9507d8

2 files changed

Lines changed: 23 additions & 3 deletions

File tree

‎sqlmesh/core/model/seed.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,9 @@ def _restore_declared_types(self, df: pd.DataFrame) -> None:
155155
Inference turns TEXT columns like `00123` into integers (dropping leading zeros) and
156156
integer columns with blanks into floats. Such columns are re-read as raw strings: TEXT
157157
columns keep them as is, and integer columns are converted to Python ints (exact and
158-
unbounded), unless a value isn't an integer literal, in which case they're left as is.
158+
unbounded), unless a non-NULL value isn't an integer literal, in which case they're left
159+
as is. Restored columns keep the NULLs of the original read, whose NA markers
160+
(`na_values`) can also match parsed numbers, unlike the raw re-read.
159161
"""
160162
import pandas as pd
161163

@@ -183,12 +185,12 @@ def _restore_declared_types(self, df: pd.DataFrame) -> None:
183185

184186
raw_df = self._read_csv(dtype=str)
185187
for name in text_columns:
186-
df[name] = raw_df[name]
188+
df[name] = raw_df[name].where(df[name].notna())
187189

188190
for name in int_columns:
189191
try:
190192
df[name] = pd.Series(
191-
[None if pd.isna(v) else int(v) for v in raw_df[name]],
193+
[None if is_na else int(v) for is_na, v in zip(df[name].isna(), raw_df[name])],
192194
index=df.index,
193195
dtype=object,
194196
)

‎tests/core/test_seed.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,24 @@ def test_declared_columns_honor_csv_settings():
175175
assert _values(df["n"]) == [1, None]
176176

177177

178+
@pytest.mark.parametrize("na_value", [1, "1"], ids=["int", "str"])
179+
def test_declared_columns_keep_numeric_na_markers_null(na_value):
180+
# pandas also matches numeric NA markers against parsed numbers (1.00 matches 1), which
181+
# the raw string re-read doesn't, so restored columns keep the NULLs of the original read
182+
seed = Seed(content="code,n,id\n1.00,1.00,10\n2,,20\n,3,30\n")
183+
settings = CsvSettings(na_values=[na_value])
184+
df = next(
185+
seed.reader(
186+
settings=settings, columns_to_types=_declared(code="text", n="int", id="int")
187+
).read()
188+
)
189+
190+
assert _values(df["code"]) == [None, "2", None]
191+
assert _values(df["n"]) == [None, None, 3]
192+
assert [type(v) for v in df["n"]] == [type(None), type(None), int]
193+
assert df["id"].to_list() == [10, 20, 30]
194+
195+
178196
def test_declared_columns_match_normalized_csv_headers():
179197
seed = Seed(content="Code,N\n001,1\n,2\n")
180198
reader = seed.reader(dialect="snowflake", columns_to_types=_declared(CODE="text", N="int"))

0 commit comments

Comments
 (0)