rebased: from main

Author: ForeverAngry

poetry.lock

@@ -84,9 +84,8 @@ pyiceberg-core = { version = "^0.4.0", optional = true }
 polars = { version = "^1.21.0", optional = true }
 thrift-sasl = { version = ">=0.4.3", optional = true }
 
-
 [tool.poetry.group.dev.dependencies]
-pytest = ">=8.3.5,<9.0.0"
+pytest = "7.4.4"
 pytest-checkdocs = "2.13.0"
 pytest-lazy-fixture = "0.6.3"
 pre-commit = "4.2.0"

pyproject.toml

@@ -2365,7 +2365,7 @@ def table_v2_with_extensive_snapshots(example_table_metadata_v2_with_extensive_s
         identifier=("database", "table"),
         metadata=table_metadata,
         metadata_location=f"{table_metadata.location}/uuid.metadata.json",
-        io=load_file_io(location=metadata_location),
+        io=load_file_io(),
         catalog=NoopCatalog("NoopCatalog"),
     )
 

tests/conftest.py

@@ -393,6 +393,22 @@ def test_string_to_boolean_literal() -> None:
     assert literal("FALSE").to(BooleanType()) == literal(False)
 
 
+def test_string_to_float_literal() -> None:
+    assert literal("3.141").to(FloatType()) == literal(3.141).to(FloatType())
+
+
+def test_string_to_float_outside_bound() -> None:
+    big_lit_str = literal(str(FloatType.max + 1.0e37))
+    assert big_lit_str.to(FloatType()) == FloatAboveMax()
+
+    small_lit_str = literal(str(FloatType.min - 1.0e37))
+    assert small_lit_str.to(FloatType()) == FloatBelowMin()
+
+
+def test_string_to_double_literal() -> None:
+    assert literal("3.141").to(DoubleType()) == literal(3.141)
+
+
 @pytest.mark.parametrize(
     "val",
     ["unknown", "off", "on", "0", "1", "y", "yes", "n", "no", "t", "f"],
@@ -744,7 +760,7 @@ def test_invalid_decimal_conversions() -> None:
 def test_invalid_string_conversions() -> None:
     assert_invalid_conversions(
         literal("abc"),
-        [FloatType(), DoubleType(), FixedType(1), BinaryType()],
+        [FixedType(1), BinaryType()],
     )
 
 

tests/expressions/test_literals.py

@@ -140,6 +140,14 @@ def test_add_hour(catalog: Catalog) -> None:
     _validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, HourTransform(), "hour_transform"))
 
 
+@pytest.mark.integration
+@pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
+def test_add_hour_string_transform(catalog: Catalog) -> None:
+    table = _table(catalog)
+    table.update_spec().add_field("event_ts", "hour", "str_hour_transform").commit()
+    _validate_new_partition_fields(table, 1000, 1, 1000, PartitionField(2, 1000, HourTransform(), "str_hour_transform"))
+
+
 @pytest.mark.integration
 @pytest.mark.parametrize("catalog", [pytest.lazy_fixture("session_catalog_hive"), pytest.lazy_fixture("session_catalog")])
 def test_add_hour_generates_default_name(catalog: Catalog) -> None:

tests/integration/test_partition_evolution.py

@@ -154,7 +154,7 @@ def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
         NestedField(field_id=4, name="col_integer", field_type=IntegerType(), required=False),
     )
 
-    with pytest.raises(CommitFailedException) as exc_info:
+    with pytest.raises(CommitFailedException, match="Requirement failed: current schema id has changed: expected 2, found 3"):
         with tbl.transaction() as tx:
             # Start a new update
             schema_update = tx.update_schema()
@@ -165,8 +165,6 @@ def test_schema_evolution_via_transaction(catalog: Catalog) -> None:
             # stage another update in the transaction
             schema_update.add_column("col_double", DoubleType()).commit()
 
-    assert "Requirement failed: current schema changed: expected id 2 != 3" in str(exc_info.value)
-
     assert tbl.schema() == Schema(
         NestedField(field_id=1, name="col_uuid", field_type=UUIDType(), required=False),
         NestedField(field_id=2, name="col_fixed", field_type=FixedType(25), required=False),

tests/integration/test_rest_schema.py

@@ -1776,3 +1776,37 @@ def test_write_optional_list(session_catalog: Catalog) -> None:
     session_catalog.load_table(identifier).append(df_2)
 
     assert len(session_catalog.load_table(identifier).scan().to_arrow()) == 4
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize("format_version", [1, 2])
+def test_evolve_and_write(
+    spark: SparkSession, session_catalog: Catalog, arrow_table_with_null: pa.Table, format_version: int
+) -> None:
+    identifier = "default.test_evolve_and_write"
+    tbl = _create_table(session_catalog, identifier, properties={"format-version": format_version}, schema=Schema())
+    other_table = session_catalog.load_table(identifier)
+
+    numbers = pa.array([1, 2, 3, 4], type=pa.int32())
+
+    with tbl.update_schema() as upd:
+        # This is not known by other_table
+        upd.add_column("id", IntegerType())
+
+    with other_table.transaction() as tx:
+        # Refreshes the underlying metadata, and the schema
+        other_table.refresh()
+        tx.append(
+            pa.Table.from_arrays(
+                [
+                    numbers,
+                ],
+                schema=pa.schema(
+                    [
+                        pa.field("id", pa.int32(), nullable=True),
+                    ]
+                ),
+            )
+        )
+
+    assert session_catalog.load_table(identifier).scan().to_arrow().column(0).combine_chunks() == numbers