Create marker column in pyarrow instead of Python list first

koenvo · koenvo · commit a088d6c6aa0f · 2025-04-17T22:27:16.000+02:00
diff --git a/pyiceberg/table/upsert_util.py b/pyiceberg/table/upsert_util.py
@@ -117,9 +117,7 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols
         )
 
         # Step 2: Prepare target index with join keys and a marker
-        target_index = target_table.select(join_cols_set).append_column(
-            MARKER_COLUMN_NAME, pa.array([True] * len(target_table), pa.bool_())
-        )
+        target_index = target_table.select(join_cols_set).append_column(MARKER_COLUMN_NAME, pa.repeat(True, len(target_table)))
 
         # Step 3: Perform a left outer join to find which rows from source exist in target
         joined = source_index.join(target_index, keys=list(join_cols_set), join_type="left outer")

Original file line number	Diff line number	Diff line change
`@@ -117,9 +117,7 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols`
`117`	`117`	`)`
`118`	`118`
`119`	`119`	`# Step 2: Prepare target index with join keys and a marker`
`120`		`- target_index = target_table.select(join_cols_set).append_column(`
`121`		`- MARKER_COLUMN_NAME, pa.array([True] * len(target_table), pa.bool_())`
`122`		`- )`
	`120`	`+ target_index = target_table.select(join_cols_set).append_column(MARKER_COLUMN_NAME, pa.repeat(True, len(target_table)))`
`123`	`121`
`124`	`122`	`# Step 3: Perform a left outer join to find which rows from source exist in target`
`125`	`123`	`joined = source_index.join(target_index, keys=list(join_cols_set), join_type="left outer")`