diff --git a/CHANGES.rst b/CHANGES.rst index 84dd336b..2a5bb585 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,6 +4,9 @@ Changes for crate Unreleased ========== +- Added JSON serialization support for Python's ``datetime.time`` type, + encoding it as an ISO 8601 string compatible with CrateDB's ``TIMETZ`` + column type. - Added gzip compression for outgoing request bodies via the ``compress`` parameter (default: ``8192`` bytes). diff --git a/docs/by-example/cursor.rst b/docs/by-example/cursor.rst index 86979fc3..ab676601 100644 --- a/docs/by-example/cursor.rst +++ b/docs/by-example/cursor.rst @@ -311,8 +311,8 @@ Python data type conversion =========================== The cursor object can optionally convert database types to native Python data -types. Currently, this is implemented for the CrateDB data types ``IP`` and -``TIMESTAMP`` on behalf of the ``DefaultTypeConverter``. +types. Currently, this is implemented for the CrateDB data types ``IP``, +``TIMESTAMP``, and ``TIMETZ`` on behalf of the ``DefaultTypeConverter``. >>> cursor = connection.cursor(converter=DefaultTypeConverter()) @@ -329,6 +329,24 @@ types. Currently, this is implemented for the CrateDB data types ``IP`` and >>> cursor.fetchone() ['foo', IPv4Address('10.10.10.1'), datetime.datetime(2022, 7, 18, 18, 10, 36, 758000, tzinfo=datetime.timezone.utc)] +CrateDB's ``TIMETZ`` type is returned over HTTP as ``[microseconds_since_midnight, tz_offset_seconds]`` +and decoded to a ``datetime.time`` object with the appropriate timezone: + + >>> cursor = connection.cursor(converter=DefaultTypeConverter()) + + >>> connection.client.set_next_response({ + ... "col_types": [20], + ... "rows":[ [ [45045000000, 0] ] ], + ... "cols":[ "t" ], + ... "rowcount":1, + ... "duration":1 + ... }) + + >>> cursor.execute('') + + >>> cursor.fetchone() + [datetime.time(12, 30, 45, tzinfo=datetime.timezone.utc)] + Custom data type conversion =========================== diff --git a/src/crate/client/converter.py b/src/crate/client/converter.py index fec80b7e..71c06aee 100644 --- a/src/crate/client/converter.py +++ b/src/crate/client/converter.py @@ -54,6 +54,24 @@ def _to_datetime(value: Optional[float]) -> Optional[dt.datetime]: return dt.datetime.fromtimestamp(value / 1e3, tz=dt.timezone.utc) +def _to_time(value: Optional[list]) -> Optional[dt.time]: + """ + Convert a CrateDB TIMETZ wire value to a Python ``datetime.time``. + + CrateDB returns TIMETZ as + ``[microseconds_since_midnight, tz_offset_seconds]`` + via the HTTP interface. + + https://docs.python.org/3/library/datetime.html#datetime.time + """ + if value is None: + return None + microseconds, tz_offset_seconds = value + tz = dt.timezone(dt.timedelta(seconds=int(tz_offset_seconds))) + t = (dt.datetime.min + dt.timedelta(microseconds=int(microseconds))).time() + return t.replace(tzinfo=tz) + + def _to_default(value: Optional[Any]) -> Optional[Any]: return value @@ -98,6 +116,7 @@ class DataType(Enum): DataType.IP: _to_ipaddress, DataType.TIMESTAMP_WITH_TZ: _to_datetime, DataType.TIMESTAMP_WITHOUT_TZ: _to_datetime, + DataType.TIME: _to_time, } diff --git a/src/crate/client/http.py b/src/crate/client/http.py index 139330ff..7492ecf2 100644 --- a/src/crate/client/http.py +++ b/src/crate/client/http.py @@ -99,6 +99,8 @@ def json_encoder(obj: t.Any) -> t.Union[int, str]: - Python's `dt.datetime` and `dt.date` types will be serialized to `int` after converting to milliseconds since epoch. + - Python's `dt.time` will be serialized to `str`, following + the ISO format. https://github.com/ijl/orjson#default https://cratedb.com/docs/crate/reference/en/latest/general/ddl/data-types.html#type-timestamp @@ -114,6 +116,8 @@ def json_encoder(obj: t.Any) -> t.Union[int, str]: delta.microseconds / 1000.0 + (delta.seconds + delta.days * 24 * 3600) * 1000.0 ) + if isinstance(obj, dt.time): + return obj.isoformat() if isinstance(obj, dt.date): return calendar.timegm(obj.timetuple()) * 1000 raise TypeError diff --git a/tests/client/test_cursor.py b/tests/client/test_cursor.py index 3888e475..ab429ac8 100644 --- a/tests/client/test_cursor.py +++ b/tests/client/test_cursor.py @@ -293,6 +293,41 @@ def test_execute_custom_converter(mocked_connection): ] +def test_execute_time_converter(mocked_connection): + """ + Verify that CrateDB's TIMETZ wire format + ``[microseconds, tz_offset_seconds]`` is decoded to a ``datetime.time`` + object by ``DefaultTypeConverter``. + """ + converter = DefaultTypeConverter() + cursor = mocked_connection.cursor(converter=converter) + response = { + "col_types": [20], + "cols": ["t"], + "rows": [ + [[45045000000, 0]], # 12:30:45 UTC + [[45045123456, 7200]], # 12:30:45.123456 +02:00 + [None], + ], + "rowcount": 3, + "duration": 1, + } + + with mock.patch.object( + mocked_connection.client, "sql", return_value=response + ): + cursor.execute("") + result = cursor.fetchall() + + assert result == [ + [datetime.time(12, 30, 45, 0, + tzinfo=datetime.timezone.utc)], + [datetime.time(12, 30, 45, 123456, + tzinfo=datetime.timezone(datetime.timedelta(hours=2)))], + [None], + ] + + def test_execute_with_converter_and_invalid_data_type(mocked_connection): converter = DefaultTypeConverter() diff --git a/tests/client/test_serialization.py b/tests/client/test_serialization.py index a5f48359..ed917c4c 100644 --- a/tests/client/test_serialization.py +++ b/tests/client/test_serialization.py @@ -125,6 +125,44 @@ def test_date_serialization(): assert result == b"1461196800000" +def test_naive_time_serialization(): + """ + Verify that a naive `datetime.time` serializes to an ISO 8601 string. + """ + data = dt.time(12, 30, 45) + result = json_dumps(data) + assert result == b'"12:30:45"' + + +def test_time_with_microseconds_serialization(): + """ + Verify that `datetime.time` with microseconds serializes correctly. + """ + data = dt.time(12, 30, 45, 123456) + result = json_dumps(data) + assert result == b'"12:30:45.123456"' + + +def test_aware_time_serialization(): + """ + Verify that a timezone-aware `datetime.time` serializes to ISO 8601 format, + including the UTC offset. + """ + data = dt.time(12, 30, 45, tzinfo=dt.timezone.utc) + result = json_dumps(data) + assert result == b'"12:30:45+00:00"' + + +def test_aware_time_with_offset_serialization(): + """ + Verify that a `datetime.time` with a non-UTC offset serializes correctly. + """ + tz = dt.timezone(dt.timedelta(hours=2)) + data = dt.time(12, 30, 45, tzinfo=tz) + result = json_dumps(data) + assert result == b'"12:30:45+02:00"' + + def test_uuid_serialization(): """ Verify that a `uuid.UUID` can be serialized. diff --git a/tests/testing/test_layer.py b/tests/testing/test_layer.py index 7270d974..250bfde1 100644 --- a/tests/testing/test_layer.py +++ b/tests/testing/test_layer.py @@ -22,13 +22,11 @@ import json import os import tempfile -import urllib from io import BytesIO from pathlib import Path from unittest import TestCase, mock import urllib3 -from verlib2 import Version import crate from crate.testing.layer import ( @@ -38,7 +36,7 @@ wait_for_http_url, ) from tests.client.settings import crate_path -from tests.conftest import download_cratedb +from tests.conftest import download_cratedb, get_crate_url class LayerUtilsTest(TestCase): @@ -86,18 +84,10 @@ def test_layer_from_uri(self): The CrateLayer can also be created by providing an URI that points to a CrateDB tarball. """ - with urllib.request.urlopen( - "https://crate.io/versions.json" - ) as response: - versions = json.loads(response.read().decode()) - version = versions["crate_testing"] - - self.assertGreaterEqual(Version(version), Version("4.5.0")) - - uri = "https://cdn.crate.io/downloads/releases/crate-{}.tar.gz".format( - version - ) - layer = CrateLayer.from_uri(uri, name="crate-by-uri", http_port=42203) + layer = CrateLayer.from_uri(get_crate_url(), + name="crate-by-uri", + http_port=42203 + ) self.assertIsInstance(layer, CrateLayer) @mock.patch.dict("os.environ", {}, clear=True)