From 883c3e9dd38a145f4fc9e054a879bf2fc895ff8a Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:18:56 +0000 Subject: [PATCH 1/2] Fix client-v2: keep the type parameters when rebuilding a type from a binary type encoding readDynamicData() rebuilds the concrete type of a value stored in a Dynamic column as a type name that is parsed back, and parent types append the child's original type name. Four branches produced a type that did not match what the server encoded: Variant wrapped its own type name twice, Nested read only the element names and left the element type encodings in the stream, Decimal and Enum returned a bare type name so their precision/scale and their constants were dropped, and the enum constant width was taken from the number of constants instead of from the type tag. Fixes: https://github.com/ClickHouse/clickhouse-java/issues/3003 --- CHANGELOG.md | 8 ++ .../internal/BinaryStreamReader.java | 33 ++++-- .../client/datatypes/DataTypeTests.java | 100 ++++++++++++++++++ 3 files changed, 131 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52c92a57a..27199a543 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,14 @@ ### Bug Fixes +- **[client-v2]** Fixed reading a `Variant`, `Nested`, `Decimal` or `Enum` value held in a `Dynamic` column. The + concrete type rebuilt from the binary type encoding did not match what the server encoded: `Variant` was wrapped + twice (so the discriminator selected the wrong element), `Nested` read only the element names and left the element + type encodings in the stream, and `Decimal`/`Enum` lost their precision and scale / their constants whenever the + value sat inside another type, so a decimal read back unscaled (`1.2500` as `12500`) and every enum value read back + as ``. The constant width of an enum is now taken from the type tag rather than from the number of + constants, which also fixes reading an `Enum16` with fewer than 128 constants and negative `Enum8` constants. + (https://github.com/ClickHouse/clickhouse-java/issues/3003) - **[client-v2]** Fixed LZ4 input streams not closing their underlying HTTP response stream. Closing an LZ4 stream returned by `QueryResponse.getInputStream()` now releases the wrapped transport stream, including after a partial read. (https://github.com/ClickHouse/clickhouse-java/issues/2985) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java index 7a244e956..43854d770 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java @@ -5,6 +5,7 @@ import com.clickhouse.data.ClickHouseColumn; import com.clickhouse.data.ClickHouseDataType; import com.clickhouse.data.ClickHouseEnum; +import com.clickhouse.data.ClickHouseUtils; import com.clickhouse.data.value.ClickHouseBitmap; import org.slf4j.Logger; import org.slf4j.helpers.NOPLogger; @@ -1459,7 +1460,9 @@ private ClickHouseColumn readDynamicData() throws IOException { case Decimal256: { int precision = readByte(); int scale = readByte(); - return ClickHouseColumn.of("v", ClickHouseDataType.binTag2Type.get(tag), false, precision, scale); + // Rendered the way the server renders a decimal type, so that a parent type encoding + // (array, tuple, map, ...) can append it and keep the precision and the scale. + return ClickHouseColumn.of("v", "Decimal(" + precision + ", " + scale + ")"); } case Dynamic: { int maxTypes = readVarInt(input); @@ -1471,16 +1474,24 @@ private ClickHouseColumn readDynamicData() throws IOException { int constants = readVarInt(input); int[] values = new int[constants]; String[] names = new String[constants]; - ClickHouseDataType enumType = constants > 127 ? ClickHouseDataType.Enum16 : ClickHouseDataType.Enum8; + // The width of the constant values is defined by the type tag, not by their number: + // Enum8 encodes them as Int8 and Enum16 as Int16. + ClickHouseDataType enumType = type == ClickHouseDataType.Enum16 ? + ClickHouseDataType.Enum16 : ClickHouseDataType.Enum8; + StringBuilder enumDef = new StringBuilder(SB_INIT_SIZE); + enumDef.append(enumType.name()).append('('); for (int i = 0; i < constants; i++) { names[i] = readString(input); - if (enumType == ClickHouseDataType.Enum8) { - values[i] = readUnsignedByte(); - } else { - values[i] = readUnsignedShortLE(); + values[i] = enumType == ClickHouseDataType.Enum8 ? readByte() : readShortLE(); + if (i > 0) { + enumDef.append(", "); } + enumDef.append('\'').append(ClickHouseUtils.escape(names[i], '\'')).append("' = ").append(values[i]); } - return new ClickHouseColumn(enumType, "v", enumType.name(), false, false, Collections.emptyList(), Collections.emptyList(), + enumDef.append(')'); + // The constants are a part of the type: a parent type encoding appends this type name, + // so it has to carry them to keep the names of the values readable. + return new ClickHouseColumn(enumType, "v", enumDef.toString(), false, false, Collections.emptyList(), Collections.emptyList(), new ClickHouseEnum(names, values)); } case FixedString: { @@ -1543,8 +1554,10 @@ private ClickHouseColumn readDynamicData() throws IOException { StringBuilder nested = new StringBuilder(SB_INIT_SIZE); nested.append("Nested("); for (int i = 0; i < size; i++) { - String name = readString(input); - nested.append(name).append(','); + // Nested is encoded as a named tuple: every element name is followed by the + // type encoding of that element, which has to be consumed here as well. + nested.append(readString(input)).append(' '); + nested.append(readDynamicData().getOriginalTypeName()).append(','); } nested.setLength(nested.length() - 1); nested.append(')'); @@ -1576,7 +1589,7 @@ private ClickHouseColumn readDynamicData() throws IOException { } variant.setLength(variant.length() - 1); variant.append(")"); - return ClickHouseColumn.of("v", "Variant(" + variant + ")"); + return ClickHouseColumn.of("v", variant.toString()); } case AggregateFunction: throw new ClientException("Aggregate functions are not supported yet"); diff --git a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java index 5e330a6a3..31739d790 100644 --- a/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java @@ -1320,6 +1320,106 @@ public void testDynamicWithNestedTypes() throws Exception { Assert.assertEquals(records.get(0).getInteger("num"), 10); } + @DataProvider(name = "dynamicParametrizedElementTypes") + public Object[][] dynamicParametrizedElementTypes() { + return new Object[][]{ + {"Decimal32(2)", "[1.25]", new String[]{"1.25"}}, + {"Decimal64(4)", "[1.25, -3.5]", new String[]{"1.2500", "-3.5000"}}, + {"Decimal(10, 2)", "[12345678.91]", new String[]{"12345678.91"}}, + {"Decimal128(6)", "[0.000001]", new String[]{"0.000001"}}, + {"Decimal256(20)", "[1.5]", new String[]{"1.50000000000000000000"}}, + {"Enum8('a' = 1, 'b' = 2)", "['b']", new String[]{"b"}}, + {"Enum8('a' = -1, 'b' = 2)", "['a']", new String[]{"a"}}, + {"Enum8('a,b' = 1, 'c\\'d' = 2)", "['a,b', 'c\\'d']", new String[]{"a,b", "c'd"}}, + {"Enum16('a' = 1000, 'b' = 2000)", "['b']", new String[]{"b"}}, + {"Enum16('a' = -1000, 'b' = 2000)", "['a']", new String[]{"a"}}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "dynamicParametrizedElementTypes") + public void testDynamicWithParametrizedElementType(String elementType, String values, String[] expected) throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + List records = client.queryAll("SELECT " + values + "::Array(" + elementType + ")::Dynamic AS v, 42::Int32 AS num"); + Object[] items = records.get(0).getObjectArray("v"); + Assert.assertEquals(items.length, expected.length); + for (int i = 0; i < expected.length; i++) { + Assert.assertEquals(String.valueOf(items[i]), expected[i]); + } + Assert.assertEquals(records.get(0).getInteger("num"), 42); + } + + @Test(groups = {"integration"}) + public void testDynamicWithEnum8WithMoreThan127Constants() throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + StringBuilder enumType = new StringBuilder("Enum8("); + for (int i = 0; i < 130; i++) { + if (i > 0) { + enumType.append(", "); + } + enumType.append('\'').append("c").append(i).append("' = ").append(i - 128); + } + enumType.append(')'); + + List records = client.queryAll("SELECT ['c0', 'c129']::Array(" + enumType + ")::Dynamic AS v, 42::Int32 AS num"); + Object[] items = records.get(0).getObjectArray("v"); + Assert.assertEquals(items.length, 2); + Assert.assertEquals(items[0].toString(), "c0"); + Assert.assertEquals(items[1].toString(), "c129"); + Assert.assertEquals(records.get(0).getInteger("num"), 42); + } + + @Test(groups = {"integration"}) + public void testDynamicWithParametrizedTypesInsideMapAndTuple() throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + List records = client.queryAll("SELECT " + + "map('k', 1.25::Decimal64(4))::Map(String, Decimal64(4))::Dynamic AS m, " + + "(1.25::Decimal64(4), 'b'::Enum8('a' = 1, 'b' = 2))::Tuple(d Decimal64(4), e Enum8('a' = 1, 'b' = 2))::Dynamic AS t, " + + "42::Int32 AS num"); + GenericRecord row = records.get(0); + Assert.assertEquals(String.valueOf(((Map) row.getObject("m")).get("k")), "1.2500"); + Object[] tuple = (Object[]) row.getObject("t"); + Assert.assertEquals(String.valueOf(tuple[0]), "1.2500"); + Assert.assertEquals(String.valueOf(tuple[1]), "b"); + Assert.assertEquals(row.getInteger("num"), 42); + } + + @Test(groups = {"integration"}) + public void testDynamicWithVariantElement() throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + // The variant elements are declared in an order that differs from the order the server encodes + // them in, so a wrongly rebuilt variant maps a discriminator to the wrong element. + List records = client.queryAll("SELECT [1, 'a']::Array(Variant(String, Int32))::Dynamic AS v, 42::Int32 AS num"); + Assert.assertEquals(records.get(0).getObjectArray("v"), new Object[]{1, "a"}); + Assert.assertEquals(records.get(0).getInteger("num"), 42); + } + + @Test(groups = {"integration"}) + public void testDynamicWithNestedElement() throws Exception { + if (isVersionMatch("(,24.8]")) { + return; + } + + List records = client.queryAll("SELECT [(1, 'x', 1.25), (2, 'y', -3.5)]" + + "::Nested(a Int32, b String, c Decimal64(4))::Dynamic AS v, 42::Int32 AS num"); + Object[] items = records.get(0).getObjectArray("v"); + Assert.assertEquals(items.length, 2); + Assert.assertEquals((Object[]) items[0], new Object[]{1, "x", new BigDecimal("1.2500")}); + Assert.assertEquals((Object[]) items[1], new Object[]{2, "y", new BigDecimal("-3.5000")}); + Assert.assertEquals(records.get(0).getInteger("num"), 42); + } + @Test(groups = {"integration"}) public void testDynamicWithFixedString() throws Exception { if (isVersionMatch("(,24.8]")) { From 2a04b2ecfbffcfa4295b1d7da8e93c3bc8e0d1e4 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Wed, 9 Sep 2026 07:41:07 +0000 Subject: [PATCH 2/2] test(client-v2): cover the Dynamic type encoding fixes with unit tests The regression tests for the Variant, Nested, Decimal and Enum fixes in readDynamicData() were integration tests. The coverage of an integration test is not reported to the quality gate, so the added lines counted as not covered. Add unit tests that decode a hand-built binary type encoding, one per defect, with a trailing guard value that detects a desynchronized stream. --- .../internal/BinaryStreamReaderTests.java | 1047 +++++++++-------- 1 file changed, 587 insertions(+), 460 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java index 521bf4ed7..0f46013b7 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java @@ -1,460 +1,587 @@ -package com.clickhouse.client.api.data_formats.internal; - -import com.clickhouse.client.api.ClientException; -import com.clickhouse.data.ClickHouseColumn; -import com.clickhouse.data.ClickHouseDataType; -import com.clickhouse.data.format.BinaryStreamUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.math.BigInteger; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.temporal.ChronoUnit; -import java.util.List; -import java.util.TimeZone; - -import org.testng.Assert; -import org.testng.annotations.BeforeClass; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -public class BinaryStreamReaderTests { - - private ZoneId tzLAX; - private ZoneId tzBER; - - @BeforeClass - void beforeClass() { - tzLAX = ZoneId.of("America/Los_Angeles"); - tzBER = ZoneId.of("Europe/Berlin"); - } - - @Test - public void testCachedByteAllocator() { - BinaryStreamReader.CachingByteBufferAllocator allocator = new BinaryStreamReader.CachingByteBufferAllocator(); - - for (int i = 0; i < 6; i++) { - int size = (int) Math.pow(2, i); - byte[] firstAllocation = allocator.allocate(size); - byte[] nextAllocation = allocator.allocate(size); - Assert.assertTrue(firstAllocation == nextAllocation, "Should be the same buffer for size " + size); - } - - for (int i = 6; i < 16; i++) { - int size = (int) Math.pow(2, i); - byte[] firstAllocation = allocator.allocate(size); - byte[] nextAllocation = allocator.allocate(size); - Assert.assertNotSame(firstAllocation, nextAllocation); - } - } - - @Test(dataProvider = "dateTestData") - void readDateZonedDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - /* - * Date is number of days since 1970-01-01 (unsigned) - * ... The date value is stored without the time zone. - */ - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDate(baos, zdt, writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDate( - new ByteArrayInputStream(bytes), - bytes, - TimeZone.getTimeZone(readTZ)), - expectedZDT); - } - - @Test(dataProvider = "dateTestData") - void readDateOffsetDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDate(baos, zdt.toOffsetDateTime(), writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDate( - new ByteArrayInputStream(bytes), - bytes, - TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), - expectedZDT.toOffsetDateTime()); - } - - @DataProvider(name = "dateTestData") - private Object[][] provideDateTestData() { - ZonedDateTime zdtLAX = ZonedDateTime.of( - 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); - ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); - return new Object[][] { - // no conversion at all - { zdtLAX, tzLAX, tzLAX, zdtLAX.truncatedTo(ChronoUnit.DAYS) }, - - // write using Berlin local date -> next day - { zdtLAX, tzBER, tzBER, zdtLAX.plusDays(1L).withZoneSameLocal(tzBER) - .truncatedTo(ChronoUnit.DAYS) }, - - // read using different time zone: local date same as original - { zdtLAX, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) - .truncatedTo(ChronoUnit.DAYS) }, - - // write using different time zone: local date same as original - { zdtBER, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) - .truncatedTo(ChronoUnit.DAYS) } - }; - - } - - @Test(dataProvider = "dateTimeTestData") - void readDateTime32ZonedDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDateTime32(baos, zdt, writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDateTime32( - new ByteArrayInputStream(bytes), - bytes, - TimeZone.getTimeZone(readTZ)), - expectedZDT.truncatedTo(ChronoUnit.SECONDS)); - } - - @Test(dataProvider = "dateTimeTestData") - void readDateTime32OffsetDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDateTime32(baos, zdt.toOffsetDateTime(), writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDateTime32( - new ByteArrayInputStream(bytes), - bytes, - TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), - expectedZDT.toOffsetDateTime().truncatedTo(ChronoUnit.SECONDS)); - } - - @Test(dataProvider = "dateTimeTestData") - void readDateTime32Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDateTime32(baos, zdt.toInstant(), writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDateTime32( - new ByteArrayInputStream(bytes), - bytes, - TimeZone.getTimeZone(readTZ)), - expectedZDT.truncatedTo(ChronoUnit.SECONDS)); - } - - @Test(dataProvider = "dateTimeTestData") - void readDateTime64Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, - ZonedDateTime expectedZDT) throws IOException - { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - SerializerUtils.writeDateTime64(baos, zdt.toInstant(), 9, writeTZ); - byte[] bytes = baos.toByteArray(); - Assert.assertEquals( - BinaryStreamReader.readDateTime64( - new ByteArrayInputStream(bytes), - bytes, - 9, - TimeZone.getTimeZone(readTZ)), - expectedZDT); - } - - @DataProvider(name = "dateTimeTestData") - private Object[][] provideDateTimeTestData() { - ZonedDateTime zdtLAX = ZonedDateTime.of( - 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); - ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); - return new Object[][] { - { zdtLAX, tzLAX, tzLAX, zdtLAX }, - { zdtLAX, tzBER, tzLAX, zdtLAX }, - { zdtLAX, tzLAX, tzBER, zdtBER }, - { zdtBER, tzLAX, tzBER, zdtBER } - }; - } - - @Test - public void testArrayValue() throws Exception { - BinaryStreamReader.ArrayValue array = new BinaryStreamReader.ArrayValue(int.class, 10); - - for (int i = 0; i < array.length(); i++) { - array.set(i, i); - } - - int[] array1 = (int[]) array.getArray(); - Object[] array2 = array.getArrayOfObjects(); - Assert.assertEquals(array1.length, array2.length); - } - - @Test - public void testDynamicSimpleAggregateFunctionConsumesWholeTypeEncoding() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - baos.write(ClickHouseDataType.SimpleAggregateFunction.getBinTag()); - BinaryStreamUtils.writeString(baos, "sum"); - BinaryStreamUtils.writeVarInt(baos, 0); - BinaryStreamUtils.writeVarInt(baos, 1); - baos.write(ClickHouseDataType.UInt64.getBinTag()); - BinaryStreamUtils.writeUnsignedInt64(baos, 42); - BinaryStreamUtils.writeInt32(baos, 4242); - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - Assert.assertEquals(reader.readValue(ClickHouseColumn.of("v", "Dynamic")), BigInteger.valueOf(42)); - Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); - } - - @Test(expectedExceptions = ClientException.class) - public void testDynamicParameterizedSimpleAggregateFunctionRejected() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - baos.write(ClickHouseDataType.SimpleAggregateFunction.getBinTag()); - BinaryStreamUtils.writeString(baos, "sum"); - BinaryStreamUtils.writeVarInt(baos, 1); - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - reader.readValue(ClickHouseColumn.of("v", "Dynamic")); - } - - @Test - public void testReadNullVariantReturnsNull() throws Exception { - ClickHouseColumn column = ClickHouseColumn.of("v", "Variant(Int32, String)"); - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(new byte[]{(byte) 0xFF}), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - Assert.assertNull(reader.readValue(column)); - } - - @Test - public void testNullableArrayValueUsesBoxedComponentType() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - BinaryStreamUtils.writeVarInt(baos, 2); - BinaryStreamUtils.writeNonNull(baos); - BinaryStreamUtils.writeFloat64(baos, 1.0); - BinaryStreamUtils.writeNonNull(baos); - BinaryStreamUtils.writeFloat64(baos, 2.0); - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( - ClickHouseColumn.of("v", "Array(Nullable(Float64))")); - - Assert.assertEquals(array.getArray().getClass().getComponentType(), Double.class); - } - - @Test - public void testNullableUnsignedArrayUsesWidenedType() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - BinaryStreamUtils.writeVarInt(baos, 2); - BinaryStreamUtils.writeNonNull(baos); - BinaryStreamUtils.writeUnsignedInt8(baos, 10); - BinaryStreamUtils.writeNonNull(baos); - BinaryStreamUtils.writeUnsignedInt8(baos, 20); - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( - ClickHouseColumn.of("v", "Array(Nullable(UInt8))")); - - Assert.assertEquals(array.getArray().getClass().getComponentType(), Short.class); - } - - @Test - public void testNullableEnumArrayUsesEnumValueType() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - BinaryStreamUtils.writeVarInt(baos, 2); - BinaryStreamUtils.writeNonNull(baos); - baos.write(1); // enum ordinal for 'a' - BinaryStreamUtils.writeNonNull(baos); - baos.write(2); // enum ordinal for 'b' - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( - ClickHouseColumn.of("v", "Array(Nullable(Enum8('a'=1,'b'=2)))")); - - Assert.assertEquals(array.getArray().getClass().getComponentType(), - BinaryStreamReader.EnumValue.class); - } - - @Test - public void testEmptyArrayTypes() throws Exception { - assertEmptyArrayComponentType("Array(UInt8)", short.class); - assertEmptyArrayComponentType("Array(Nullable(UInt8))", Short.class); - assertEmptyArrayComponentType("Array(String)", String.class); - assertEmptyArrayComponentType("Array(Nullable(String))", String.class); - assertEmptyArrayComponentType("Array(Enum8('a'=1))", BinaryStreamReader.EnumValue.class); - assertEmptyArrayComponentType("Array(Nullable(Enum8('a'=1)))", BinaryStreamReader.EnumValue.class); - assertEmptyArrayComponentType("Array(Variant(Int32, String))", Object.class); - assertEmptyArrayComponentType("Array(Array(String))", BinaryStreamReader.ArrayValue.class); - } - - private void assertEmptyArrayComponentType(String columnType, Class expectedComponentType) throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - BinaryStreamUtils.writeVarInt(baos, 0); - - BinaryStreamReader reader = new BinaryStreamReader( - new ByteArrayInputStream(baos.toByteArray()), - TimeZone.getTimeZone("UTC"), - null, - new BinaryStreamReader.CachingByteBufferAllocator(), - false, - null, - false); - - BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( - ClickHouseColumn.of("v", columnType)); - - Assert.assertEquals(array.getArray().getClass().getComponentType(), expectedComponentType, "Failed for " + columnType); - } - - // Native-format QBit column payloads captured from ClickHouse 26.5.1 (the block header stripped, - // leaving the Tuple(FixedString) bit-plane bytes readQBitNative consumes) paired with the vector the - // server encoded. Pins the decode against the server's real layout as a unit test, because the QBit - // integration tests skip on the coverage build's older server. - @DataProvider(name = "qbitNativeGoldenBytes") - public static Object[][] qbitNativeGoldenBytes() { - return new Object[][] { - {"Float32 dim 3", "QBit(Float32, 3)", 1, - "0206010101010101010404000000000000000000000000000000000000000000", - false, new Object[] {new float[] {1f, -2f, 3.5f}}}, - {"Float32 dim 8 (full single-byte plane)", "QBit(Float32, 8)", 1, - "02fe010101010181796454000000000000000000000000000000000000000000", - false, new Object[] {new float[] {1f, -2f, 3.5f, 4f, 5f, 6f, 7f, 8f}}}, - {"Float32 dim 10 (two-byte plane, partial last byte)", "QBit(Float32, 10)", 1, - "02aa03fe000100010001000100010381007900640254010000000000000000000000000000000000000000000000000000000000000000000000000000000000", - false, new Object[] {new float[] {1f, -2f, 3.5f, -4f, 5f, -6f, 7f, -8f, 9f, -10f}}}, - {"Float32 dim 1", "QBit(Float32, 1)", 1, - "0001000000000100000001000100010000000000000000000000000000000000", - false, new Object[] {new float[] {42.5f}}}, - {"Float64 dim 3 (64 bit planes)", "QBit(Float64, 3)", 1, - "02060101010101010101010104040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - true, new Object[] {new double[] {1d, -2d, 3.5d}}}, - {"BFloat16 dim 8 (16 bit planes)", "QBit(BFloat16, 8)", 1, - "00fe01010101e1995500000000000000", - false, new Object[] {new float[] {1f, 2f, 4f, 8f, 16f, 32f, 64f, 128f}}}, - {"Float32 dim 3, three rows (column-major slicing)", "QBit(Float32, 3)", 3, - "000000060707010000010000010000010000010000010006010701040401000201000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - false, new Object[] { - new float[] {1f, 2f, 3f}, new float[] {4f, 5f, 6f}, new float[] {7f, 8f, 9f}}}, - }; - } - - @Test(dataProvider = "qbitNativeGoldenBytes") - public void testReadQBitNativeDecodesGoldenBytes(String label, String columnType, int nRows, - String columnDataHex, boolean isDouble, Object[] expectedRows) throws Exception { - List rows = qbitReader(fromHex(columnDataHex)) - .readQBitNative(ClickHouseColumn.of("vec", columnType), nRows); - - Assert.assertEquals(rows.size(), nRows, label); - for (int r = 0; r < nRows; r++) { - Object vector = ((BinaryStreamReader.ArrayValue) rows.get(r)).getArray(); - if (isDouble) { - Assert.assertEquals((double[]) vector, (double[]) expectedRows[r], label + " row " + r); - } else { - Assert.assertEquals((float[]) vector, (float[]) expectedRows[r], label + " row " + r); - } - } - } - - @Test - public void testReadQBitNativeDecodesSpecialFloatValues() throws Exception { - // NaN, +Inf, -Inf and -0.0 must survive the bit-plane transpose bit-for-bit. - float[] vector = (float[]) ((BinaryStreamReader.ArrayValue) qbitReader(fromHex( - "0c07070707070707070100000000000000000000000000000000000000000000")) - .readQBitNative(ClickHouseColumn.of("vec", "QBit(Float32, 4)"), 1).get(0)).getArray(); - - float[] expected = {Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, -0.0f}; - Assert.assertEquals(vector.length, expected.length); - for (int j = 0; j < expected.length; j++) { - Assert.assertEquals(Float.floatToRawIntBits(vector[j]), Float.floatToRawIntBits(expected[j]), - "element " + j); - } - } - - @Test - public void testReadQBitNativeRejectsUnsupportedElementType() { - // readQBitNative reconstructs only float element types; a non-float element is rejected by its own - // guard (the Native reader also filters these earlier), which fires before any read. - ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Int8, 3)"); - ClientException ex = Assert.expectThrows(ClientException.class, - () -> qbitReader(new byte[0]).readQBitNative(column, 1)); - Assert.assertTrue(ex.getMessage().contains("Float32"), - "Expected an unsupported-element-type message, got: " + ex.getMessage()); - } - - @Test - public void testReadQBitNativeRejectsIntOverflowPlaneSize() { - // nRows * ceil(dimension/8) per bit plane must not overflow a 32-bit int; the reader rejects it - // before allocating or reading, so an empty stream is sufficient (200000 -> 25000 bytes/plane, - // 90000 rows -> 2_250_000_000 bytes > Integer.MAX_VALUE). - ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Float32, 200000)"); - ClientException ex = Assert.expectThrows(ClientException.class, - () -> qbitReader(new byte[0]).readQBitNative(column, 90000)); - Assert.assertTrue(ex.getMessage().contains("too large"), - "Expected an overflow rejection message, got: " + ex.getMessage()); - } - - private static BinaryStreamReader qbitReader(byte[] columnData) { - return new BinaryStreamReader(new ByteArrayInputStream(columnData), TimeZone.getTimeZone("UTC"), - null, new BinaryStreamReader.CachingByteBufferAllocator(), false, null, false); - } - - private static byte[] fromHex(String hex) { - byte[] out = new byte[hex.length() / 2]; - for (int i = 0; i < out.length; i++) { - out[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16); - } - return out; - } -} +package com.clickhouse.client.api.data_formats.internal; + +import com.clickhouse.client.api.ClientException; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.format.BinaryStreamUtils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.TimeZone; + +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class BinaryStreamReaderTests { + + private ZoneId tzLAX; + private ZoneId tzBER; + + @BeforeClass + void beforeClass() { + tzLAX = ZoneId.of("America/Los_Angeles"); + tzBER = ZoneId.of("Europe/Berlin"); + } + + @Test + public void testCachedByteAllocator() { + BinaryStreamReader.CachingByteBufferAllocator allocator = new BinaryStreamReader.CachingByteBufferAllocator(); + + for (int i = 0; i < 6; i++) { + int size = (int) Math.pow(2, i); + byte[] firstAllocation = allocator.allocate(size); + byte[] nextAllocation = allocator.allocate(size); + Assert.assertTrue(firstAllocation == nextAllocation, "Should be the same buffer for size " + size); + } + + for (int i = 6; i < 16; i++) { + int size = (int) Math.pow(2, i); + byte[] firstAllocation = allocator.allocate(size); + byte[] nextAllocation = allocator.allocate(size); + Assert.assertNotSame(firstAllocation, nextAllocation); + } + } + + @Test(dataProvider = "dateTestData") + void readDateZonedDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + /* + * Date is number of days since 1970-01-01 (unsigned) + * ... The date value is stored without the time zone. + */ + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDate(baos, zdt, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDate( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT); + } + + @Test(dataProvider = "dateTestData") + void readDateOffsetDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDate(baos, zdt.toOffsetDateTime(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDate( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), + expectedZDT.toOffsetDateTime()); + } + + @DataProvider(name = "dateTestData") + private Object[][] provideDateTestData() { + ZonedDateTime zdtLAX = ZonedDateTime.of( + 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); + ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); + return new Object[][] { + // no conversion at all + { zdtLAX, tzLAX, tzLAX, zdtLAX.truncatedTo(ChronoUnit.DAYS) }, + + // write using Berlin local date -> next day + { zdtLAX, tzBER, tzBER, zdtLAX.plusDays(1L).withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) }, + + // read using different time zone: local date same as original + { zdtLAX, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) }, + + // write using different time zone: local date same as original + { zdtBER, tzLAX, tzBER, zdtLAX.withZoneSameLocal(tzBER) + .truncatedTo(ChronoUnit.DAYS) } + }; + + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32ZonedDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32OffsetDateTime(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt.toOffsetDateTime(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)).toOffsetDateTime(), + expectedZDT.toOffsetDateTime().truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime32Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime32(baos, zdt.toInstant(), writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime32( + new ByteArrayInputStream(bytes), + bytes, + TimeZone.getTimeZone(readTZ)), + expectedZDT.truncatedTo(ChronoUnit.SECONDS)); + } + + @Test(dataProvider = "dateTimeTestData") + void readDateTime64Instant(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, + ZonedDateTime expectedZDT) throws IOException + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + SerializerUtils.writeDateTime64(baos, zdt.toInstant(), 9, writeTZ); + byte[] bytes = baos.toByteArray(); + Assert.assertEquals( + BinaryStreamReader.readDateTime64( + new ByteArrayInputStream(bytes), + bytes, + 9, + TimeZone.getTimeZone(readTZ)), + expectedZDT); + } + + @DataProvider(name = "dateTimeTestData") + private Object[][] provideDateTimeTestData() { + ZonedDateTime zdtLAX = ZonedDateTime.of( + 2025, 7, 20, 22, 23, 1, 232323232, tzLAX); + ZonedDateTime zdtBER = zdtLAX.withZoneSameInstant(tzBER); + return new Object[][] { + { zdtLAX, tzLAX, tzLAX, zdtLAX }, + { zdtLAX, tzBER, tzLAX, zdtLAX }, + { zdtLAX, tzLAX, tzBER, zdtBER }, + { zdtBER, tzLAX, tzBER, zdtBER } + }; + } + + @Test + public void testArrayValue() throws Exception { + BinaryStreamReader.ArrayValue array = new BinaryStreamReader.ArrayValue(int.class, 10); + + for (int i = 0; i < array.length(); i++) { + array.set(i, i); + } + + int[] array1 = (int[]) array.getArray(); + Object[] array2 = array.getArrayOfObjects(); + Assert.assertEquals(array1.length, array2.length); + } + + @Test + public void testDynamicSimpleAggregateFunctionConsumesWholeTypeEncoding() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.SimpleAggregateFunction.getBinTag()); + BinaryStreamUtils.writeString(baos, "sum"); + BinaryStreamUtils.writeVarInt(baos, 0); + BinaryStreamUtils.writeVarInt(baos, 1); + baos.write(ClickHouseDataType.UInt64.getBinTag()); + BinaryStreamUtils.writeUnsignedInt64(baos, 42); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("v", "Dynamic")), BigInteger.valueOf(42)); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + @Test(expectedExceptions = ClientException.class) + public void testDynamicParameterizedSimpleAggregateFunctionRejected() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.SimpleAggregateFunction.getBinTag()); + BinaryStreamUtils.writeString(baos, "sum"); + BinaryStreamUtils.writeVarInt(baos, 1); + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + reader.readValue(ClickHouseColumn.of("v", "Dynamic")); + } + + @Test + public void testReadNullVariantReturnsNull() throws Exception { + ClickHouseColumn column = ClickHouseColumn.of("v", "Variant(Int32, String)"); + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(new byte[]{(byte) 0xFF}), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + Assert.assertNull(reader.readValue(column)); + } + + @Test + public void testNullableArrayValueUsesBoxedComponentType() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeNonNull(baos); + BinaryStreamUtils.writeFloat64(baos, 1.0); + BinaryStreamUtils.writeNonNull(baos); + BinaryStreamUtils.writeFloat64(baos, 2.0); + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( + ClickHouseColumn.of("v", "Array(Nullable(Float64))")); + + Assert.assertEquals(array.getArray().getClass().getComponentType(), Double.class); + } + + @Test + public void testNullableUnsignedArrayUsesWidenedType() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeNonNull(baos); + BinaryStreamUtils.writeUnsignedInt8(baos, 10); + BinaryStreamUtils.writeNonNull(baos); + BinaryStreamUtils.writeUnsignedInt8(baos, 20); + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( + ClickHouseColumn.of("v", "Array(Nullable(UInt8))")); + + Assert.assertEquals(array.getArray().getClass().getComponentType(), Short.class); + } + + @Test + public void testNullableEnumArrayUsesEnumValueType() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeNonNull(baos); + baos.write(1); // enum ordinal for 'a' + BinaryStreamUtils.writeNonNull(baos); + baos.write(2); // enum ordinal for 'b' + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( + ClickHouseColumn.of("v", "Array(Nullable(Enum8('a'=1,'b'=2)))")); + + Assert.assertEquals(array.getArray().getClass().getComponentType(), + BinaryStreamReader.EnumValue.class); + } + + @Test + public void testEmptyArrayTypes() throws Exception { + assertEmptyArrayComponentType("Array(UInt8)", short.class); + assertEmptyArrayComponentType("Array(Nullable(UInt8))", Short.class); + assertEmptyArrayComponentType("Array(String)", String.class); + assertEmptyArrayComponentType("Array(Nullable(String))", String.class); + assertEmptyArrayComponentType("Array(Enum8('a'=1))", BinaryStreamReader.EnumValue.class); + assertEmptyArrayComponentType("Array(Nullable(Enum8('a'=1)))", BinaryStreamReader.EnumValue.class); + assertEmptyArrayComponentType("Array(Variant(Int32, String))", Object.class); + assertEmptyArrayComponentType("Array(Array(String))", BinaryStreamReader.ArrayValue.class); + } + + private void assertEmptyArrayComponentType(String columnType, Class expectedComponentType) throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + BinaryStreamUtils.writeVarInt(baos, 0); + + BinaryStreamReader reader = new BinaryStreamReader( + new ByteArrayInputStream(baos.toByteArray()), + TimeZone.getTimeZone("UTC"), + null, + new BinaryStreamReader.CachingByteBufferAllocator(), + false, + null, + false); + + BinaryStreamReader.ArrayValue array = (BinaryStreamReader.ArrayValue) reader.readValue( + ClickHouseColumn.of("v", columnType)); + + Assert.assertEquals(array.getArray().getClass().getComponentType(), expectedComponentType, "Failed for " + columnType); + } + + // Native-format QBit column payloads captured from ClickHouse 26.5.1 (the block header stripped, + // leaving the Tuple(FixedString) bit-plane bytes readQBitNative consumes) paired with the vector the + // server encoded. Pins the decode against the server's real layout as a unit test, because the QBit + // integration tests skip on the coverage build's older server. + @DataProvider(name = "qbitNativeGoldenBytes") + public static Object[][] qbitNativeGoldenBytes() { + return new Object[][] { + {"Float32 dim 3", "QBit(Float32, 3)", 1, + "0206010101010101010404000000000000000000000000000000000000000000", + false, new Object[] {new float[] {1f, -2f, 3.5f}}}, + {"Float32 dim 8 (full single-byte plane)", "QBit(Float32, 8)", 1, + "02fe010101010181796454000000000000000000000000000000000000000000", + false, new Object[] {new float[] {1f, -2f, 3.5f, 4f, 5f, 6f, 7f, 8f}}}, + {"Float32 dim 10 (two-byte plane, partial last byte)", "QBit(Float32, 10)", 1, + "02aa03fe000100010001000100010381007900640254010000000000000000000000000000000000000000000000000000000000000000000000000000000000", + false, new Object[] {new float[] {1f, -2f, 3.5f, -4f, 5f, -6f, 7f, -8f, 9f, -10f}}}, + {"Float32 dim 1", "QBit(Float32, 1)", 1, + "0001000000000100000001000100010000000000000000000000000000000000", + false, new Object[] {new float[] {42.5f}}}, + {"Float64 dim 3 (64 bit planes)", "QBit(Float64, 3)", 1, + "02060101010101010101010104040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + true, new Object[] {new double[] {1d, -2d, 3.5d}}}, + {"BFloat16 dim 8 (16 bit planes)", "QBit(BFloat16, 8)", 1, + "00fe01010101e1995500000000000000", + false, new Object[] {new float[] {1f, 2f, 4f, 8f, 16f, 32f, 64f, 128f}}}, + {"Float32 dim 3, three rows (column-major slicing)", "QBit(Float32, 3)", 3, + "000000060707010000010000010000010000010000010006010701040401000201000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + false, new Object[] { + new float[] {1f, 2f, 3f}, new float[] {4f, 5f, 6f}, new float[] {7f, 8f, 9f}}}, + }; + } + + @Test(dataProvider = "qbitNativeGoldenBytes") + public void testReadQBitNativeDecodesGoldenBytes(String label, String columnType, int nRows, + String columnDataHex, boolean isDouble, Object[] expectedRows) throws Exception { + List rows = qbitReader(fromHex(columnDataHex)) + .readQBitNative(ClickHouseColumn.of("vec", columnType), nRows); + + Assert.assertEquals(rows.size(), nRows, label); + for (int r = 0; r < nRows; r++) { + Object vector = ((BinaryStreamReader.ArrayValue) rows.get(r)).getArray(); + if (isDouble) { + Assert.assertEquals((double[]) vector, (double[]) expectedRows[r], label + " row " + r); + } else { + Assert.assertEquals((float[]) vector, (float[]) expectedRows[r], label + " row " + r); + } + } + } + + @Test + public void testReadQBitNativeDecodesSpecialFloatValues() throws Exception { + // NaN, +Inf, -Inf and -0.0 must survive the bit-plane transpose bit-for-bit. + float[] vector = (float[]) ((BinaryStreamReader.ArrayValue) qbitReader(fromHex( + "0c07070707070707070100000000000000000000000000000000000000000000")) + .readQBitNative(ClickHouseColumn.of("vec", "QBit(Float32, 4)"), 1).get(0)).getArray(); + + float[] expected = {Float.NaN, Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, -0.0f}; + Assert.assertEquals(vector.length, expected.length); + for (int j = 0; j < expected.length; j++) { + Assert.assertEquals(Float.floatToRawIntBits(vector[j]), Float.floatToRawIntBits(expected[j]), + "element " + j); + } + } + + @Test + public void testReadQBitNativeRejectsUnsupportedElementType() { + // readQBitNative reconstructs only float element types; a non-float element is rejected by its own + // guard (the Native reader also filters these earlier), which fires before any read. + ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Int8, 3)"); + ClientException ex = Assert.expectThrows(ClientException.class, + () -> qbitReader(new byte[0]).readQBitNative(column, 1)); + Assert.assertTrue(ex.getMessage().contains("Float32"), + "Expected an unsupported-element-type message, got: " + ex.getMessage()); + } + + @Test + public void testReadQBitNativeRejectsIntOverflowPlaneSize() { + // nRows * ceil(dimension/8) per bit plane must not overflow a 32-bit int; the reader rejects it + // before allocating or reading, so an empty stream is sufficient (200000 -> 25000 bytes/plane, + // 90000 rows -> 2_250_000_000 bytes > Integer.MAX_VALUE). + ClickHouseColumn column = ClickHouseColumn.of("vec", "QBit(Float32, 200000)"); + ClientException ex = Assert.expectThrows(ClientException.class, + () -> qbitReader(new byte[0]).readQBitNative(column, 90000)); + Assert.assertTrue(ex.getMessage().contains("too large"), + "Expected an overflow rejection message, got: " + ex.getMessage()); + } + + @Test + public void testDynamicDecimalKeepsPrecisionAndScaleInsideArray() throws Exception { + // 0x2x : the precision and the scale belong to the type, so a parent + // encoding (here Array) must keep them, otherwise the value is read back unscaled. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Array.getBinTag()); + baos.write(ClickHouseDataType.Decimal64.getBinTag()); + BinaryStreamUtils.writeInt8(baos, 18); // precision + BinaryStreamUtils.writeInt8(baos, 4); // scale + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeInt64(baos, 12500); + BinaryStreamUtils.writeInt64(baos, -35000); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] values = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + Assert.assertEquals(values, new Object[]{new BigDecimal("1.2500"), new BigDecimal("-3.5000")}); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + @Test + public void testDynamicEnum8KeepsConstantsAndReadsThemSigned() throws Exception { + // The constants belong to the type as well, and an Enum8 constant is a signed byte. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Array.getBinTag()); + baos.write(ClickHouseDataType.Enum8.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeString(baos, "a'b"); + BinaryStreamUtils.writeInt8(baos, -1); + BinaryStreamUtils.writeString(baos, "c"); + BinaryStreamUtils.writeInt8(baos, 2); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeInt8(baos, -1); + BinaryStreamUtils.writeInt8(baos, 2); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] values = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + assertEnumValue(values[0], "a'b", -1); + assertEnumValue(values[1], "c", 2); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + @Test + public void testDynamicEnum16TakesConstantWidthFromTag() throws Exception { + // The width of a constant is defined by the tag (Enum16 -> Int16), not by the number of + // constants: an Enum16 with less than 128 constants desynchronizes the stream otherwise. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Array.getBinTag()); + baos.write(ClickHouseDataType.Enum16.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeString(baos, "a"); + BinaryStreamUtils.writeInt16(baos, -1000); + BinaryStreamUtils.writeString(baos, "b"); + BinaryStreamUtils.writeInt16(baos, 2000); + BinaryStreamUtils.writeVarInt(baos, 1); + BinaryStreamUtils.writeInt16(baos, -1000); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] values = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + assertEnumValue(values[0], "a", -1000); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + @Test + public void testDynamicNestedConsumesElementTypeEncodings() throws Exception { + // Nested is encoded as a named tuple: every element name is followed by the type encoding + // of that element, which has to be consumed as well - including the parameters of that + // element type (here the precision and the scale of the decimal). + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Nested.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeString(baos, "a"); + baos.write(ClickHouseDataType.Decimal64.getBinTag()); + BinaryStreamUtils.writeInt8(baos, 18); // precision + BinaryStreamUtils.writeInt8(baos, 4); // scale + BinaryStreamUtils.writeString(baos, "b"); + baos.write(ClickHouseDataType.String.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 1); + BinaryStreamUtils.writeInt64(baos, 12500); + BinaryStreamUtils.writeString(baos, "x"); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] rows = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + Assert.assertEquals(rows.length, 1); + Assert.assertEquals((Object[]) rows[0], new Object[]{new BigDecimal("1.2500"), "x"}); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + @Test + public void testDynamicVariantIsNotWrappedTwice() throws Exception { + // A variant rebuilt as Variant(Variant(...)) has a single element, so the discriminator + // selects the wrong alternative. + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + baos.write(ClickHouseDataType.Array.getBinTag()); + baos.write(ClickHouseDataType.Variant.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + baos.write(ClickHouseDataType.Int32.getBinTag()); + baos.write(ClickHouseDataType.String.getBinTag()); + BinaryStreamUtils.writeVarInt(baos, 2); + BinaryStreamUtils.writeInt8(baos, 1); // discriminator of String + BinaryStreamUtils.writeString(baos, "a"); + BinaryStreamUtils.writeInt8(baos, 0); // discriminator of Int32 + BinaryStreamUtils.writeInt32(baos, 1); + BinaryStreamUtils.writeInt32(baos, 4242); + + BinaryStreamReader reader = dynamicReader(baos); + Object[] values = ((BinaryStreamReader.ArrayValue) reader.readValue(ClickHouseColumn.of("v", "Dynamic"))).getArrayOfObjects(); + Assert.assertEquals(values, new Object[]{"a", 1}); + Assert.assertEquals(reader.readValue(ClickHouseColumn.of("guard", "Int32")), Integer.valueOf(4242)); + } + + private static void assertEnumValue(Object actual, String expectedName, int expectedValue) { + BinaryStreamReader.EnumValue value = (BinaryStreamReader.EnumValue) actual; + Assert.assertEquals(value.getName(), expectedName); + Assert.assertEquals(value.intValue(), expectedValue); + } + + private static BinaryStreamReader dynamicReader(ByteArrayOutputStream columnData) { + return new BinaryStreamReader(new ByteArrayInputStream(columnData.toByteArray()), + TimeZone.getTimeZone("UTC"), null, new BinaryStreamReader.CachingByteBufferAllocator(), + false, null, false); + } + + private static BinaryStreamReader qbitReader(byte[] columnData) { + return new BinaryStreamReader(new ByteArrayInputStream(columnData), TimeZone.getTimeZone("UTC"), + null, new BinaryStreamReader.CachingByteBufferAllocator(), false, null, false); + } + + private static byte[] fromHex(String hex) { + byte[] out = new byte[hex.length() / 2]; + for (int i = 0; i < out.length; i++) { + out[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16); + } + return out; + } +}