MOARRR CODE

Author: Fokko

pyiceberg/avro/resolver.py

@@ -290,7 +290,7 @@ def struct(self, file_schema: StructType, record_struct: Optional[IcebergType],
                 # There is a default value
                 if file_field.write_default is not None:
                     # The field is not in the record, but there is a write default value
-                    results.append((None, DefaultWriter(writer=writer, value=file_field.write_default)))  # type: ignore
+                    results.append((None, DefaultWriter(writer=writer, value=file_field.write_default)))
                 elif file_field.required:
                     raise ValueError(f"Field is required, and there is no write default: {file_field}")
             else:

pyiceberg/conversions.py

@@ -503,27 +503,47 @@ def _(_: Union[IntegerType, LongType], val: int) -> int:
 
 
 @from_json.register(DateType)
-def _(_: DateType, val: str) -> date:
+def _(_: DateType, val: Union[str, int, date]) -> date:
     """JSON date is string encoded."""
-    return days_to_date(date_str_to_days(val))
+    if isinstance(val, str):
+        val = date_str_to_days(val)
+    if isinstance(val, int):
+        return days_to_date(val)
+    else:
+        return val
 
 
 @from_json.register(TimeType)
-def _(_: TimeType, val: str) -> time:
+def _(_: TimeType, val: Union[str, int, time]) -> time:
     """JSON ISO8601 string into Python time."""
-    return micros_to_time(time_str_to_micros(val))
+    if isinstance(val, str):
+        val = time_str_to_micros(val)
+    if isinstance(val, int):
+        return micros_to_time(val)
+    else:
+        return val
 
 
 @from_json.register(TimestampType)
-def _(_: PrimitiveType, val: str) -> datetime:
+def _(_: PrimitiveType, val: Union[str, int, datetime]) -> datetime:
     """JSON ISO8601 string into Python datetime."""
-    return micros_to_timestamp(timestamp_to_micros(val))
+    if isinstance(val, str):
+        val = timestamp_to_micros(val)
+    if isinstance(val, int):
+        return micros_to_timestamp(val)
+    else:
+        return val
 
 
 @from_json.register(TimestamptzType)
-def _(_: TimestamptzType, val: str) -> datetime:
+def _(_: TimestamptzType, val: Union[str, int, datetime]) -> datetime:
     """JSON ISO8601 string into Python datetime."""
-    return micros_to_timestamptz(timestamptz_to_micros(val))
+    if isinstance(val, str):
+        val = timestamptz_to_micros(val)
+    if isinstance(val, int):
+        return micros_to_timestamptz(val)
+    else:
+        return val
 
 
 @from_json.register(FloatType)
@@ -540,20 +560,26 @@ def _(_: StringType, val: str) -> str:
 
 
 @from_json.register(FixedType)
-def _(t: FixedType, val: str) -> bytes:
+def _(t: FixedType, val: Union[str, bytes]) -> bytes:
     """JSON hexadecimal encoded string into bytes."""
-    b = codecs.decode(val.encode(UTF8), "hex")
+    if isinstance(val, str):
+        b = codecs.decode(val.encode(UTF8), "hex")
 
-    if len(t) != len(b):
-        raise ValueError(f"FixedType has length {len(t)}, which is different from the value: {len(b)}")
+        if len(t) != len(b):
+            raise ValueError(f"FixedType has length {len(t)}, which is different from the value: {len(b)}")
 
-    return b
+        return b
+    else:
+        return val
 
 
 @from_json.register(BinaryType)
-def _(_: BinaryType, val: str) -> bytes:
+def _(_: BinaryType, val: Union[bytes, str]) -> bytes:
     """JSON hexadecimal encoded string into bytes."""
-    return codecs.decode(val.encode(UTF8), "hex")
+    if isinstance(val, str):
+        return codecs.decode(val.encode(UTF8), "hex")
+    else:
+        return val
 
 
 @from_json.register(DecimalType)
@@ -563,6 +589,11 @@ def _(_: DecimalType, val: str) -> Decimal:
 
 
 @from_json.register(UUIDType)
-def _(_: UUIDType, val: str) -> uuid.UUID:
+def _(_: UUIDType, val: Union[str, bytes, uuid.UUID]) -> uuid.UUID:
     """Convert JSON string into Python UUID."""
-    return uuid.UUID(val)
+    if isinstance(val, str):
+        return uuid.UUID(val)
+    elif isinstance(val, bytes):
+        return uuid.UUID(bytes=val)
+    else:
+        return val

pyiceberg/expressions/literals.py

@@ -23,7 +23,7 @@
 
 import struct
 from abc import ABC, abstractmethod
-from datetime import date, datetime
+from datetime import date, datetime, time
 from decimal import ROUND_HALF_UP, Decimal
 from functools import singledispatchmethod
 from math import isnan
@@ -54,6 +54,7 @@
     datetime_to_micros,
     micros_to_days,
     time_str_to_micros,
+    time_to_micros,
     timestamp_to_micros,
     timestamptz_to_micros,
 )
@@ -152,6 +153,8 @@ def literal(value: L) -> Literal[L]:
         return TimestampLiteral(datetime_to_micros(value))  # type: ignore
     elif isinstance(value, date):
         return DateLiteral(date_to_days(value))  # type: ignore
+    elif isinstance(value, time):
+        return TimeLiteral(time_to_micros(value))
     else:
         raise TypeError(f"Invalid literal value: {repr(value)}")
 

pyiceberg/table/update/schema.py

@@ -398,11 +398,11 @@ def _set_column_default_value(self, path: Union[str, Tuple[str, ...]], default_v
             except ValueError as e:
                 raise ValueError(f"Invalid default value: {e}") from e
 
-        if field.required and default_value != field.write_default:
+        if field.required and default_value == field.write_default:
             # if the change is a noop, allow it even if allowIncompatibleChanges is false
             return
 
-        if not self._allow_incompatible_changes and field.required and default_value is not None:
+        if not self._allow_incompatible_changes and field.required and default_value is None:
             raise ValueError("Cannot change change default-value of a required column to None")
 
         if field.field_id in self._deletes:
@@ -729,19 +729,35 @@ def struct(self, struct: StructType, field_results: List[Optional[IcebergType]])
             name = field.name
             doc = field.doc
             required = field.required
+            write_default = field.write_default
 
             # There is an update
             if update := self._updates.get(field.field_id):
                 name = update.name
                 doc = update.doc
                 required = update.required
-
-            if field.name == name and field.field_type == result_type and field.required == required and field.doc == doc:
+                write_default = update.write_default
+
+            if (
+                field.name == name
+                and field.field_type == result_type
+                and field.required == required
+                and field.doc == doc
+                and field.write_default == write_default
+            ):
                 new_fields.append(field)
             else:
                 has_changes = True
                 new_fields.append(
-                    NestedField(field_id=field.field_id, name=name, field_type=result_type, required=required, doc=doc)
+                    NestedField(
+                        field_id=field.field_id,
+                        name=name,
+                        field_type=result_type,
+                        required=required,
+                        doc=doc,
+                        initial_default=field.initial_default,
+                        write_default=write_default,
+                    )
                 )
 
         if has_changes:

pyiceberg/types.py

@@ -35,22 +35,24 @@
 import re
 from functools import cached_property
 from typing import (
+    Annotated,
     Any,
     ClassVar,
     Dict,
     Literal,
     Optional,
-    Tuple, Annotated,
+    Tuple,
 )
 
 from pydantic import (
+    BeforeValidator,
     Field,
     PrivateAttr,
     SerializeAsAny,
     model_serializer,
-    model_validator, PlainSerializer, BeforeValidator,
+    model_validator,
 )
-from pydantic_core.core_schema import ValidatorFunctionWrapHandler, SerializationInfo, ValidationInfo
+from pydantic_core.core_schema import ValidationInfo, ValidatorFunctionWrapHandler
 
 from pyiceberg.exceptions import ValidationError
 from pyiceberg.typedef import IcebergBaseModel, IcebergRootModel, L, TableVersion
@@ -289,24 +291,21 @@ def __eq__(self, other: Any) -> bool:
         return self.root == other.root if isinstance(other, DecimalType) else False
 
 
-# def _serialize_default_value(v: Any, context: SerializationInfo) -> Any:
-#     from pyiceberg.conversions import to_json, from_json
-#     return v
 def _deserialize_default_value(v: Any, context: ValidationInfo) -> Any:
-    if context.mode != 'python':
-        if v is not None:
-            from pyiceberg.conversions import from_json
-            return from_json(context.data.get("field_type"), v)
-        else:
-            return None
+    if v is not None:
+        from pyiceberg.conversions import from_json
+
+        return from_json(context.data.get("field_type"), v)
     else:
-        return v
+        return None
+
 
-# PlainSerializer(_serialize_default_value, return_type=Any),
 DefaultValue = Annotated[
-    Any, BeforeValidator(_deserialize_default_value)
+    L,
+    BeforeValidator(_deserialize_default_value),
 ]
 
+
 class NestedField(IcebergType):
     """Represents a field of a struct, a map key, a map value, or a list element.
 
@@ -335,8 +334,8 @@ class NestedField(IcebergType):
     field_type: SerializeAsAny[IcebergType] = Field(alias="type")
     required: bool = Field(default=False)
     doc: Optional[str] = Field(default=None, repr=False)
-    initial_default: DefaultValue = Field(alias="initial-default", default=None, repr=False)
-    write_default: DefaultValue = Field(alias="write-default", default=None, repr=False)  # type: ignore
+    initial_default: Optional[DefaultValue] = Field(alias="initial-default", default=None, repr=False)  # type: ignore
+    write_default: Optional[DefaultValue] = Field(alias="write-default", default=None, repr=False)  # type: ignore
 
     def __init__(
         self,
@@ -360,6 +359,26 @@ def __init__(
         data["write-default"] = data["write-default"] if "write-default" in data else write_default
         super().__init__(**data)
 
+    @model_serializer()
+    def serialize_model(self) -> Dict[str, Any]:
+        from pyiceberg.conversions import to_json
+
+        fields = {
+            "id": self.field_id,
+            "name": self.name,
+            "type": self.field_type,
+            "required": self.required,
+        }
+
+        if self.doc is not None:
+            fields["doc"] = self.doc
+        if self.initial_default is not None:
+            fields["initial-default"] = to_json(self.field_type, self.initial_default)
+        if self.write_default is not None:
+            fields["write-default"] = to_json(self.field_type, self.write_default)
+
+        return fields
+
     def __str__(self) -> str:
         """Return the string representation of the NestedField class."""
         doc = "" if not self.doc else f" ({self.doc})"

pyiceberg/utils/schema_conversion.py

@@ -530,7 +530,7 @@ def field(self, field: NestedField, field_result: AvroType) -> AvroType:
         }
 
         if field.write_default is not None:
-            result["default"] = field.write_default  # type: ignore
+            result["default"] = field.write_default
         elif field.optional:
             result["default"] = None
 

tests/integration/test_rest_schema.py

@@ -24,7 +24,6 @@
 
 from pyiceberg.catalog import Catalog, load_catalog
 from pyiceberg.exceptions import CommitFailedException, NoSuchTableError, ValidationError
-from pyiceberg.expressions import literal  # type: ignore
 from pyiceberg.partitioning import PartitionField, PartitionSpec
 from pyiceberg.schema import Schema, prune_columns
 from pyiceberg.table import Table, TableProperties
@@ -1100,25 +1099,25 @@ def test_add_required_column(catalog: Catalog) -> None:
 @pytest.mark.parametrize(
     "iceberg_type, default_value, write_default",
     [
-        # (BooleanType(), True, False),
-        # (IntegerType(), 123, 456),
-        # (LongType(), 123, 456),
-        # (FloatType(), 19.25, 22.27),
-        # (DoubleType(), 19.25, 22.27),
+        (BooleanType(), True, False),
+        (IntegerType(), 123, 456),
+        (LongType(), 123, 456),
+        (FloatType(), 19.25, 22.27),
+        (DoubleType(), 19.25, 22.27),
         (DecimalType(10, 2), Decimal("19.25"), Decimal("22.27")),
-        # (DecimalType(10, 2), Decimal("19.25"), Decimal("22.27")),
-        # (StringType(), "abc", "def"),
-        # (DateType(), date(1990, 3, 1), date(1991, 3, 1)),
-        # (TimeType(), time(19, 25, 22), time(22, 25, 22)),
-        # (TimestampType(), datetime(1990, 5, 1, 22, 1, 1), datetime(2000, 5, 1, 22, 1, 1)),
-        # (
-        #     TimestamptzType(),
-        #     datetime(1990, 5, 1, 22, 1, 1, tzinfo=timezone.utc),
-        #     datetime(2000, 5, 1, 22, 1, 1, tzinfo=timezone.utc),
-        # ),
-        # (BinaryType(), b"123", b"456"),
-        # (FixedType(4), b"1234", b"5678"),
-        # (UUIDType(), UUID(int=0x12345678123456781234567812345678), UUID(int=0x32145678123456781234567812345678)),
+        (DecimalType(10, 2), Decimal("19.25"), Decimal("22.27")),
+        (StringType(), "abc", "def"),
+        (DateType(), date(1990, 3, 1), date(1991, 3, 1)),
+        (TimeType(), time(19, 25, 22), time(22, 25, 22)),
+        (TimestampType(), datetime(1990, 5, 1, 22, 1, 1), datetime(2000, 5, 1, 22, 1, 1)),
+        (
+            TimestamptzType(),
+            datetime(1990, 5, 1, 22, 1, 1, tzinfo=timezone.utc),
+            datetime(2000, 5, 1, 22, 1, 1, tzinfo=timezone.utc),
+        ),
+        (BinaryType(), b"123", b"456"),
+        (FixedType(4), b"1234", b"5678"),
+        (UUIDType(), UUID(int=0x12345678123456781234567812345678), UUID(int=0x32145678123456781234567812345678)),
     ],
 )
 def test_initial_default_all_columns(
@@ -1132,17 +1131,15 @@ def test_initial_default_all_columns(
     tx.commit()
 
     field = table.schema().find_field(1)
-    physical_type = literal(default_value).to(iceberg_type).value
-    assert field.initial_default == physical_type
-    assert field.write_default == physical_type
+    assert field.initial_default == default_value
+    assert field.write_default == default_value
 
     with table.update_schema() as tx:
         tx.set_default_value("data", write_default)
 
     field = table.schema().find_field(1)
-    write_physical_type = literal(default_value).to(iceberg_type).value
-    assert field.initial_default == physical_type
-    assert field.write_default == write_physical_type
+    assert field.initial_default == default_value
+    assert field.write_default == write_default
 
 
 @pytest.mark.integration

tests/test_types.py

@@ -225,11 +225,9 @@ def test_nested_field() -> None:
     assert str(field_var) == str(eval(repr(field_var)))
     assert field_var == pickle.loads(pickle.dumps(field_var))
 
-    with pytest.raises(pydantic_core.ValidationError) as exc_info:
+    with pytest.raises(pydantic_core.ValidationError, match=".*validation errors for NestedField.*"):
         _ = (NestedField(1, "field", StringType(), required=True, write_default=(1, "a", True)),)  # type: ignore
 
-    assert "validation errors for NestedField" in str(exc_info.value)
-
 
 @pytest.mark.parametrize("input_index,input_type", non_parameterized_types)
 @pytest.mark.parametrize("check_index,check_type", non_parameterized_types)

tests/utils/test_manifest.py

@@ -416,7 +416,7 @@ def test_write_manifest(
 
         data_file = manifest_entry.data_file
 
-        assert data_file.content is DataFileContent.DATA
+        assert data_file.content == DataFileContent.DATA
         assert (
             data_file.file_path
             == "/home/iceberg/warehouse/nyc/taxis_partitioned/data/VendorID=null/00000-633-d8a4223e-dc97-45a1-86e1-adaba6e8abd7-00001.parquet"