support fsspec s3 addressing_style

Author: fcfangcc

pyiceberg/io/__init__.py

@@ -62,6 +62,7 @@
 S3_PROXY_URI = "s3.proxy-uri"
 S3_CONNECT_TIMEOUT = "s3.connect-timeout"
 S3_REQUEST_TIMEOUT = "s3.request-timeout"
+S3_ADDRESSING_STYLE = "s3.addressing-style"
 S3_SIGNER = "s3.signer"
 S3_SIGNER_URI = "s3.signer.uri"
 S3_SIGNER_ENDPOINT = "s3.signer.endpoint"

pyiceberg/io/fsspec.py

@@ -67,6 +67,7 @@
     HF_ENDPOINT,
     HF_TOKEN,
     S3_ACCESS_KEY_ID,
+    S3_ADDRESSING_STYLE,
     S3_ANONYMOUS,
     S3_CONNECT_TIMEOUT,
     S3_ENDPOINT,
@@ -168,6 +169,9 @@ def _s3(properties: Properties) -> AbstractFileSystem:
     if request_timeout := properties.get(S3_REQUEST_TIMEOUT):
         config_kwargs["read_timeout"] = float(request_timeout)
 
+    if addressing_style := properties.get(S3_ADDRESSING_STYLE):
+        config_kwargs["s3"] = {"addressing_style": addressing_style}
+
     if s3_anonymous := properties.get(S3_ANONYMOUS):
         anon = strtobool(s3_anonymous)
     else:

tests/io/test_fsspec.py

@@ -306,6 +306,36 @@ def test_fsspec_s3_session_properties() -> None:
         )
 
 
+def test_fsspec_s3_session_properties_with_addressing_style() -> None:
+    session_properties: Properties = {
+        "s3.addressing-style": "virtual",
+        "s3.endpoint": "http://localhost:9000",
+        "s3.access-key-id": "admin",
+        "s3.secret-access-key": "password",
+        "s3.region": "us-east-1",
+        "s3.session-token": "s3.session-token",
+        **UNIFIED_AWS_SESSION_PROPERTIES,
+    }
+
+    with mock.patch("s3fs.S3FileSystem") as mock_s3fs:
+        s3_fileio = FsspecFileIO(properties=session_properties)
+        filename = str(uuid.uuid4())
+
+        s3_fileio.new_input(location=f"s3://warehouse/{filename}")
+
+        mock_s3fs.assert_called_with(
+            anon=False,
+            client_kwargs={
+                "endpoint_url": "http://localhost:9000",
+                "aws_access_key_id": "admin",
+                "aws_secret_access_key": "password",
+                "region_name": "us-east-1",
+                "aws_session_token": "s3.session-token",
+            },
+            config_kwargs={"s3": {"addressing_style": "virtual"}},
+        )
+
+
 def test_fsspec_s3_session_properties_with_anonymous() -> None:
     session_properties: Properties = {
         "s3.anonymous": "true",