From e6adf5f69ada128c6092b9fffc586de69a9d2cca Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:37:46 +0000 Subject: [PATCH 1/8] Fix client-v2: detect the codec of a compressed response instead of assuming LZ4 ClickHouse 26.9 switched the default codec of the HTTP compress=1 framing to ZSTD(3), while the response reader asserted the LZ4 method byte of every block, so every compressed read failed with "Invalid LZ4 magic byte: '-112'". The framing is self-describing, so the reader now takes the codec from the block header and decompresses LZ4, ZSTD and uncompressed blocks. A block of an unknown codec is still rejected, which keeps the fallback that reads an unframed error body. zstd-jni becomes a required dependency of client-v2 and is shaded into the all artifacts. Fixes: https://github.com/ClickHouse/clickhouse-java/issues/3105 --- CHANGELOG.md | 8 + clickhouse-jdbc/pom.xml | 3 +- client-v2/pom.xml | 13 +- .../com/clickhouse/client/api/Client.java | 2 + .../internal/ClickHouseLZ4InputStream.java | 71 +++++++- .../api/internal/HttpAPIClientHelper.java | 4 +- .../ClickHouseLZ4InputStreamTest.java | 163 ++++++++++++++++++ docs/features.md | 2 +- 8 files changed, 250 insertions(+), 16 deletions(-) create mode 100644 client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 9eff8d530..b99ef3c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,6 +137,14 @@ ### Bug Fixes +- **[client-v2]** Fixed every compressed read failing with `Invalid LZ4 magic byte: '-112'` against ClickHouse `26.9` + and later. The server picks the codec of the `compress=1` framing and now uses `ZSTD(3)` by default, while the + response reader asserted the LZ4 method byte of every block, so any query answered with a compressed body died + before the first row was parsed and no setting could restore the old codec. The reader now takes the codec from the + block header, which is what makes the framing self-describing, and decompresses LZ4, ZSTD and uncompressed blocks — + including a body that mixes them. A block of an unknown codec is still rejected, so a response that is not framed at + all is reported as before. `zstd-jni` became a required dependency of `client-v2` and is shaded into the `all` + artifacts. (https://github.com/ClickHouse/clickhouse-java/issues/3105) - **[jdbc-v2]** Fixed `DatabaseMetaData#getTables` reporting `TABLE_TYPE = TABLE` for a table with the `BigQuery` engine (present in `system.table_engines` since ClickHouse `26.8`). The engine was missing from the engine-to-table-type mapping, so it fell back to the default `TABLE`, and `getTables(..., types = {"REMOTE TABLE"})` diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml index c4c56727f..495db0ade 100644 --- a/clickhouse-jdbc/pom.xml +++ b/clickhouse-jdbc/pom.xml @@ -58,10 +58,11 @@ lz4-java ${lz4.version} + com.github.luben zstd-jni - provided org.slf4j diff --git a/client-v2/pom.xml b/client-v2/pom.xml index c1b4e92d2..40869a710 100644 --- a/client-v2/pom.xml +++ b/client-v2/pom.xml @@ -58,6 +58,13 @@ ${lz4.version} + + + com.github.luben + zstd-jni + + org.apache.commons commons-compress @@ -188,12 +195,6 @@ 5.19.0 test - - com.github.luben - zstd-jni - 1.5.7-6 - test - org.bouncycastle bcprov-jdk18on diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index dbe29a268..786fbfe5e 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -695,6 +695,8 @@ public Builder setLZ4UncompressedBufferSize(int size) { /** * Disable native compression. If set to true then native compression will be disabled. * If from some reason the native compressor is not working then it can be disabled. + * Applies to LZ4 only: a response the server compressed with ZSTD is always read with the + * native library of zstd-jni, because the server picks the codec of the response. * @param disable * @return */ diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java index 8550fb6c2..77e3b16e8 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java @@ -4,6 +4,7 @@ import com.clickhouse.data.ClickHouseByteUtils; import com.clickhouse.data.ClickHouseCityHash; import com.clickhouse.data.ClickHouseUtils; +import com.github.luben.zstd.Zstd; import net.jpountz.lz4.LZ4FastDecompressor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,6 +14,11 @@ import java.io.InputStream; import java.nio.ByteBuffer; +/** + * Reads the framed output of the ClickHouse HTTP {@code compress=1} interface. Each block is + * self-describing: its header carries the compression method the server used, so the codec is + * detected per block instead of being assumed. LZ4, ZSTD and uncompressed blocks are supported. + */ public class ClickHouseLZ4InputStream extends InputStream { private static Logger LOG = LoggerFactory.getLogger(ClickHouseLZ4InputStream.class); @@ -27,7 +33,7 @@ public class ClickHouseLZ4InputStream extends InputStream { public ClickHouseLZ4InputStream(InputStream in, LZ4FastDecompressor decompressor, int bufferSize) { super(); - LOG.debug("Using LZ4 decompressor with buffer size {}", bufferSize); + LOG.debug("Reading compressed response with buffer size {}", bufferSize); this.decompressor = decompressor; this.in = in; this.buffer = ByteBuffer.allocate(bufferSize); @@ -66,6 +72,8 @@ public int read(byte[] b, int off, int len) throws IOException { static final byte MAGIC = (byte) 0x82; + static final byte MAGIC_ZSTD = (byte) 0x90; + static final byte MAGIC_NONE = (byte) 0x02; static final int HEADER_LENGTH = 25; final byte[] headerBuff = new byte[HEADER_LENGTH]; @@ -107,9 +115,10 @@ private int refill() throws IOException { return -1; } - if (headerBuff[16] != MAGIC) { - // 1 byte - 0x82 (shows this is LZ4) - throw new ClientException("Invalid LZ4 magic byte: '" + headerBuff[16] + "'"); + // 1 byte - compression method (0x82 LZ4, 0x90 ZSTD, 0x02 uncompressed) + final byte method = headerBuff[16]; + if (method != MAGIC && method != MAGIC_ZSTD && method != MAGIC_NONE) { + throw new ClientException("Invalid compression method byte: '" + method + "'"); } // 4 bytes - size of the compressed data including 9 bytes of the header @@ -118,8 +127,14 @@ private int refill() throws IOException { int uncompressedSize = getInt32(headerBuff, 21); int offset = 9; + if (compressedSizeWithHeader < offset || uncompressedSize < 0) { + throw new ClientException(ClickHouseUtils.format( + "Corrupted stream: block declares {0} compressed and {1} uncompressed bytes", + compressedSizeWithHeader, uncompressedSize)); + } + final byte[] block = new byte[compressedSizeWithHeader]; - block[0] = MAGIC; + block[0] = method; setInt32(block, 1, compressedSizeWithHeader); setInt32(block, 5, uncompressedSize); // compressed data: compressed_size - 9 bytes @@ -138,12 +153,56 @@ private int refill() throws IOException { if (buffer.capacity() < uncompressedSize) { buffer = ByteBuffer.allocate(uncompressedSize); } - decompressor.decompress(ByteBuffer.wrap(block), offset, buffer, 0, uncompressedSize); + decompress(method, block, offset, remaining, uncompressedSize); buffer.position(0); buffer.limit(uncompressedSize); return uncompressedSize; } + /** + * Decompresses a single block into {@link #buffer} with the codec the block header declares. + * + * @param method compression method byte of the block + * @param block block, including its 9 bytes of header + * @param offset offset of the compressed data in the block + * @param compressedSize size of the compressed data + * @param uncompressedSize size of the data after decompression + */ + private void decompress(byte method, byte[] block, int offset, int compressedSize, int uncompressedSize) { + switch (method) { + case MAGIC: + decompressor.decompress(ByteBuffer.wrap(block), offset, buffer, 0, uncompressedSize); + break; + case MAGIC_ZSTD: + long decompressedSize; + try { + decompressedSize = Zstd.decompressByteArray(buffer.array(), buffer.arrayOffset(), + uncompressedSize, block, offset, compressedSize); + } catch (LinkageError e) { + // the server picks the codec of the response, so ZSTD cannot be avoided by configuration + throw new ClientException("Server compressed the response with ZSTD but the native library of " + + "zstd-jni is not available on this platform", e); + } + if (Zstd.isError(decompressedSize)) { + throw new ClientException("Failed to decompress ZSTD block: " + + Zstd.getErrorName(decompressedSize)); + } else if (decompressedSize != uncompressedSize) { + throw new ClientException(ClickHouseUtils.format( + "Corrupted stream: decompressed {0} bytes while {1} were expected", + decompressedSize, uncompressedSize)); + } + break; + default: // MAGIC_NONE + if (compressedSize != uncompressedSize) { + throw new ClientException(ClickHouseUtils.format( + "Corrupted stream: uncompressed block holds {0} bytes while {1} were expected", + compressedSize, uncompressedSize)); + } + System.arraycopy(block, offset, buffer.array(), buffer.arrayOffset(), uncompressedSize); + break; + } + } + /** * Read int32 Little Endian * @param bytes diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 54d53eec9..9b303a960 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -447,7 +447,7 @@ private ServerException readNotClickHouseError(HttpEntity httpEntity, String que } break; } catch (ClientException e) { - // Invalid LZ4 Magic + // response body is not the framed output of the compress=1 interface if (body instanceof ClickHouseLZ4InputStream) { ClickHouseLZ4InputStream stream = (ClickHouseLZ4InputStream) body; body = stream.getInputStream(); @@ -478,7 +478,7 @@ private static ServerException readClickHouseError(HttpEntity httpEntity, int se try { rBytes = body.read(buffer); } catch (ClientException e) { - // Invalid LZ4 Magic + // response body is not the framed output of the compress=1 interface if (body instanceof ClickHouseLZ4InputStream) { ClickHouseLZ4InputStream stream = (ClickHouseLZ4InputStream) body; body = stream.getInputStream(); diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java new file mode 100644 index 000000000..f01a06469 --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java @@ -0,0 +1,163 @@ +package com.clickhouse.client.api.internal; + +import com.clickhouse.client.api.ClientException; +import com.clickhouse.data.ClickHouseCityHash; +import com.github.luben.zstd.Zstd; +import net.jpountz.lz4.LZ4Factory; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +public class ClickHouseLZ4InputStreamTest { + + private static final int BUFFER_SIZE = 8192; + + @Test(groups = {"unit"}, dataProvider = "compressionMethodProvider") + public void testReadsBlockOfEveryCompressionMethod(byte method) throws IOException { + byte[] payload = payload(); + + Assert.assertEquals(readFully(frame(method, payload), BUFFER_SIZE), payload); + } + + @Test(groups = {"unit"}, dataProvider = "compressionMethodProvider") + public void testReadsBlockLargerThanInitialBuffer(byte method) throws IOException { + byte[] payload = payload(); + + Assert.assertEquals(readFully(frame(method, payload), 16), payload); + } + + @DataProvider(name = "compressionMethodProvider") + public Object[][] compressionMethodProvider() { + return new Object[][]{ + {ClickHouseLZ4InputStream.MAGIC}, + {ClickHouseLZ4InputStream.MAGIC_ZSTD}, + {ClickHouseLZ4InputStream.MAGIC_NONE}, + }; + } + + @Test(groups = {"unit"}) + public void testReadsConsecutiveBlocksOfDifferentMethods() throws IOException { + byte[] first = "first block\n".getBytes(StandardCharsets.UTF_8); + byte[] second = "second block\n".getBytes(StandardCharsets.UTF_8); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + stream.write(frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, first)); + stream.write(frame(ClickHouseLZ4InputStream.MAGIC, second)); + + byte[] expected = new byte[first.length + second.length]; + System.arraycopy(first, 0, expected, 0, first.length); + System.arraycopy(second, 0, expected, first.length, second.length); + Assert.assertEquals(readFully(stream.toByteArray(), BUFFER_SIZE), expected); + } + + @Test(groups = {"unit"}) + public void testRejectsUnknownCompressionMethod() { + byte[] frame = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload()); + frame[16] = (byte) 0x42; + + ClientException e = Assert.expectThrows(ClientException.class, () -> readFully(frame, BUFFER_SIZE)); + Assert.assertTrue(e.getMessage().contains("Invalid compression method byte"), e.getMessage()); + } + + @Test(groups = {"unit"}) + public void testRejectsBlockOfImpossibleSize() { + byte[] frame = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload()); + ClickHouseLZ4InputStream.setInt32(frame, 17, 4); + + ClientException e = Assert.expectThrows(ClientException.class, () -> readFully(frame, BUFFER_SIZE)); + Assert.assertTrue(e.getMessage().contains("Corrupted stream"), e.getMessage()); + } + + @Test(groups = {"unit"}) + public void testKeepsUnframedBodyReadableThroughHeaderBuffer() throws IOException { + byte[] body = "Code: 62. DB::Exception: Syntax error: failed at position 1\n" + .getBytes(StandardCharsets.UTF_8); + + try (ClickHouseLZ4InputStream in = new ClickHouseLZ4InputStream(new ByteArrayInputStream(body), + LZ4Factory.fastestInstance().fastDecompressor(), BUFFER_SIZE)) { + Assert.expectThrows(ClientException.class, () -> in.read(new byte[64])); + Assert.assertEquals(in.getHeaderBuffer(), + Arrays.copyOf(body, ClickHouseLZ4InputStream.HEADER_LENGTH)); + } + } + + @Test(groups = {"unit"}) + public void testRejectsCorruptedBlock() { + byte[] frame = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload()); + frame[frame.length - 1] ^= 0xFF; + + ClientException e = Assert.expectThrows(ClientException.class, () -> readFully(frame, BUFFER_SIZE)); + Assert.assertTrue(e.getMessage().contains("checksum mismatch"), e.getMessage()); + } + + private static byte[] payload() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 100; i++) { + sb.append(i).append('\t').append("value-").append(i).append('\n'); + } + return sb.toString().getBytes(StandardCharsets.UTF_8); + } + + private static byte[] readFully(byte[] frames, int bufferSize) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (InputStream in = new ClickHouseLZ4InputStream(new ByteArrayInputStream(frames), + LZ4Factory.fastestInstance().fastDecompressor(), bufferSize)) { + byte[] chunk = new byte[64]; + int read; + while ((read = in.read(chunk)) != -1) { + out.write(chunk, 0, read); + } + } + return out.toByteArray(); + } + + /** + * Builds a block the way the server frames the output of the HTTP {@code compress=1} + * interface: 16 bytes of CityHash128 checksum over the block, then a 9 byte header holding the + * compression method, the compressed size (header included) and the uncompressed size, then + * the compressed data. + */ + private static byte[] frame(byte method, byte[] data) { + byte[] compressed; + switch (method) { + case ClickHouseLZ4InputStream.MAGIC: + byte[] lz4 = new byte[LZ4Factory.fastestInstance().fastCompressor().maxCompressedLength(data.length)]; + int lz4Length = LZ4Factory.fastestInstance().fastCompressor() + .compress(data, 0, data.length, lz4, 0, lz4.length); + compressed = new byte[lz4Length]; + System.arraycopy(lz4, 0, compressed, 0, lz4Length); + break; + case ClickHouseLZ4InputStream.MAGIC_ZSTD: + compressed = Zstd.compress(data, 3); + break; + default: + compressed = data; + break; + } + + byte[] block = new byte[9 + compressed.length]; + block[0] = method; + ClickHouseLZ4InputStream.setInt32(block, 1, block.length); + ClickHouseLZ4InputStream.setInt32(block, 5, data.length); + System.arraycopy(compressed, 0, block, 9, compressed.length); + + long[] checksum = ClickHouseCityHash.cityHash128(block, 0, block.length); + byte[] frame = new byte[16 + block.length]; + setInt64(frame, 0, checksum[0]); + setInt64(frame, 8, checksum[1]); + System.arraycopy(block, 0, frame, 16, block.length); + return frame; + } + + private static void setInt64(byte[] bytes, int offset, long value) { + for (int i = 0; i < 8; i++) { + bytes[offset + i] = (byte) (0xFF & (value >> (8 * i))); + } + } +} diff --git a/docs/features.md b/docs/features.md index 308f694de..8df78cace 100644 --- a/docs/features.md +++ b/docs/features.md @@ -31,7 +31,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Session handling: Supports client-wide and per-operation HTTP sessions, operation-level session overrides, runtime updates of client `session_id`, and server-side session validation through `session_check`. - Metadata discovery: Loads table schemas from table names or queries and allows schema registration for typed read/write operations. - Server information loading: Can refresh server version, current user, and server time zone information. -- Compression support: Supports response compression, ClickHouse LZ4 request/response compression, HTTP content compression, and caller-supplied precompressed insert bodies. +- Compression support: Supports response compression, ClickHouse LZ4 request compression, HTTP content compression, and caller-supplied precompressed insert bodies. A compressed response is read with the codec of each block, so the server is free to answer with LZ4, ZSTD or uncompressed blocks. - Retry behavior: Can retry failed operations for configured failure causes and retry limits. - Client-side request cancellation: `Client.cancelTransportRequest(String queryId)` aborts the in-flight HTTP request and its IO for the operation started with the given query id. It requires the caller to set the query id in operation settings, is best-effort (it cancels client-side IO but the result is not guaranteed), and does not issue a server-side `KILL QUERY` - the server stops the query on its own once the client disconnects. A cancelled operation that is being retried stops instead of issuing another request, also when the cancellation lands between two attempts (for example from `DataStreamWriter#onRetry()`). - Metrics and observability: Exposes client/server operation metrics and optionally integrates connection-pool gauges with Micrometer. From 0ce58b016a774e56b54fefe45cc14617c5e6703f Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:16:03 +0000 Subject: [PATCH 2/8] Cover the failure paths of the block decoder and report them consistently The new negative paths of the block decoder had no test, and writing them exposed two defects in them: - ClickHouseUtils.format delegates to String.format, so the MessageFormat style placeholders of the three new messages were printed literally and the byte counts they carry were lost. - Zstd.decompressByteArray throws ZstdException instead of returning an error code, so the Zstd.isError branch was unreachable and a corrupted ZSTD block escaped as a ZstdException while every other failure of the decoder is reported as a ClientException. The corrupted frame tests now run from one data provider that covers an unknown compression method, an impossible block size, a negative uncompressed size, a checksum mismatch, a block the ZSTD codec rejects and a ZSTD or uncompressed block whose declared size does not hold. --- .../internal/ClickHouseLZ4InputStream.java | 14 +-- .../ClickHouseLZ4InputStreamTest.java | 103 +++++++++++++----- 2 files changed, 83 insertions(+), 34 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java index 77e3b16e8..aff39fcee 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java @@ -5,6 +5,7 @@ import com.clickhouse.data.ClickHouseCityHash; import com.clickhouse.data.ClickHouseUtils; import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdException; import net.jpountz.lz4.LZ4FastDecompressor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -129,7 +130,7 @@ private int refill() throws IOException { int offset = 9; if (compressedSizeWithHeader < offset || uncompressedSize < 0) { throw new ClientException(ClickHouseUtils.format( - "Corrupted stream: block declares {0} compressed and {1} uncompressed bytes", + "Corrupted stream: block declares %s compressed and %s uncompressed bytes", compressedSizeWithHeader, uncompressedSize)); } @@ -178,24 +179,23 @@ private void decompress(byte method, byte[] block, int offset, int compressedSiz try { decompressedSize = Zstd.decompressByteArray(buffer.array(), buffer.arrayOffset(), uncompressedSize, block, offset, compressedSize); + } catch (ZstdException e) { + throw new ClientException("Failed to decompress ZSTD block: " + e.getMessage(), e); } catch (LinkageError e) { // the server picks the codec of the response, so ZSTD cannot be avoided by configuration throw new ClientException("Server compressed the response with ZSTD but the native library of " + "zstd-jni is not available on this platform", e); } - if (Zstd.isError(decompressedSize)) { - throw new ClientException("Failed to decompress ZSTD block: " - + Zstd.getErrorName(decompressedSize)); - } else if (decompressedSize != uncompressedSize) { + if (decompressedSize != uncompressedSize) { throw new ClientException(ClickHouseUtils.format( - "Corrupted stream: decompressed {0} bytes while {1} were expected", + "Corrupted stream: decompressed %s bytes while %s were expected", decompressedSize, uncompressedSize)); } break; default: // MAGIC_NONE if (compressedSize != uncompressedSize) { throw new ClientException(ClickHouseUtils.format( - "Corrupted stream: uncompressed block holds {0} bytes while {1} were expected", + "Corrupted stream: uncompressed block holds %s bytes while %s were expected", compressedSize, uncompressedSize)); } System.arraycopy(block, offset, buffer.array(), buffer.arrayOffset(), uncompressedSize); diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java index f01a06469..5283d4ad8 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java @@ -3,6 +3,7 @@ import com.clickhouse.client.api.ClientException; import com.clickhouse.data.ClickHouseCityHash; import com.github.luben.zstd.Zstd; +import com.github.luben.zstd.ZstdException; import net.jpountz.lz4.LZ4Factory; import org.testng.Assert; import org.testng.annotations.DataProvider; @@ -19,6 +20,18 @@ public class ClickHouseLZ4InputStreamTest { private static final int BUFFER_SIZE = 8192; + private static final int CHECKSUM_LENGTH = 16; + + private static final int BLOCK_HEADER_LENGTH = 9; + + private static final int METHOD_OFFSET = CHECKSUM_LENGTH; + + private static final int COMPRESSED_SIZE_OFFSET = CHECKSUM_LENGTH + 1; + + private static final int UNCOMPRESSED_SIZE_OFFSET = CHECKSUM_LENGTH + 5; + + private static final int DATA_OFFSET = CHECKSUM_LENGTH + BLOCK_HEADER_LENGTH; + @Test(groups = {"unit"}, dataProvider = "compressionMethodProvider") public void testReadsBlockOfEveryCompressionMethod(byte method) throws IOException { byte[] payload = payload(); @@ -56,22 +69,57 @@ public void testReadsConsecutiveBlocksOfDifferentMethods() throws IOException { Assert.assertEquals(readFully(stream.toByteArray(), BUFFER_SIZE), expected); } - @Test(groups = {"unit"}) - public void testRejectsUnknownCompressionMethod() { - byte[] frame = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload()); - frame[16] = (byte) 0x42; - + @Test(groups = {"unit"}, dataProvider = "corruptedFrameProvider") + public void testRejectsCorruptedFrame(byte[] frame, String expectedMessage, Class expectedCause) { ClientException e = Assert.expectThrows(ClientException.class, () -> readFully(frame, BUFFER_SIZE)); - Assert.assertTrue(e.getMessage().contains("Invalid compression method byte"), e.getMessage()); + Assert.assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); + if (expectedCause == null) { + Assert.assertNull(e.getCause()); + } else { + Assert.assertTrue(expectedCause.isInstance(e.getCause()), String.valueOf(e.getCause())); + } } - @Test(groups = {"unit"}) - public void testRejectsBlockOfImpossibleSize() { - byte[] frame = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload()); - ClickHouseLZ4InputStream.setInt32(frame, 17, 4); + @DataProvider(name = "corruptedFrameProvider") + public Object[][] corruptedFrameProvider() { + byte[] payload = payload(); - ClientException e = Assert.expectThrows(ClientException.class, () -> readFully(frame, BUFFER_SIZE)); - Assert.assertTrue(e.getMessage().contains("Corrupted stream"), e.getMessage()); + byte[] unknownMethod = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); + unknownMethod[METHOD_OFFSET] = (byte) 0x42; + + byte[] impossibleCompressedSize = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); + ClickHouseLZ4InputStream.setInt32(impossibleCompressedSize, COMPRESSED_SIZE_OFFSET, 4); + + byte[] negativeUncompressedSize = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); + ClickHouseLZ4InputStream.setInt32(negativeUncompressedSize, UNCOMPRESSED_SIZE_OFFSET, -1); + + byte[] corruptedPayload = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); + corruptedPayload[corruptedPayload.length - 1] ^= 0xFF; + + // the ZSTD frame of the block loses its magic, so the codec itself rejects the data + byte[] undecompressibleZstdBlock = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); + Arrays.fill(undecompressibleZstdBlock, DATA_OFFSET, DATA_OFFSET + 4, (byte) 0); + reseal(undecompressibleZstdBlock); + + byte[] shortZstdBlock = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); + ClickHouseLZ4InputStream.setInt32(shortZstdBlock, UNCOMPRESSED_SIZE_OFFSET, payload.length + 8); + reseal(shortZstdBlock); + + byte[] shortUncompressedBlock = frame(ClickHouseLZ4InputStream.MAGIC_NONE, payload); + ClickHouseLZ4InputStream.setInt32(shortUncompressedBlock, UNCOMPRESSED_SIZE_OFFSET, payload.length - 1); + reseal(shortUncompressedBlock); + + return new Object[][]{ + {unknownMethod, "Invalid compression method byte", null}, + {impossibleCompressedSize, "block declares 4 compressed", null}, + {negativeUncompressedSize, "-1 uncompressed bytes", null}, + {corruptedPayload, "checksum mismatch", null}, + {undecompressibleZstdBlock, "Failed to decompress ZSTD block", ZstdException.class}, + {shortZstdBlock, "decompressed " + payload.length + " bytes while " + + (payload.length + 8) + " were expected", null}, + {shortUncompressedBlock, "uncompressed block holds " + payload.length + " bytes while " + + (payload.length - 1) + " were expected", null}, + }; } @Test(groups = {"unit"}) @@ -87,15 +135,6 @@ public void testKeepsUnframedBodyReadableThroughHeaderBuffer() throws IOExceptio } } - @Test(groups = {"unit"}) - public void testRejectsCorruptedBlock() { - byte[] frame = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload()); - frame[frame.length - 1] ^= 0xFF; - - ClientException e = Assert.expectThrows(ClientException.class, () -> readFully(frame, BUFFER_SIZE)); - Assert.assertTrue(e.getMessage().contains("checksum mismatch"), e.getMessage()); - } - private static byte[] payload() { StringBuilder sb = new StringBuilder(); for (int i = 0; i < 100; i++) { @@ -141,18 +180,28 @@ private static byte[] frame(byte method, byte[] data) { break; } - byte[] block = new byte[9 + compressed.length]; + byte[] block = new byte[BLOCK_HEADER_LENGTH + compressed.length]; block[0] = method; ClickHouseLZ4InputStream.setInt32(block, 1, block.length); ClickHouseLZ4InputStream.setInt32(block, 5, data.length); - System.arraycopy(compressed, 0, block, 9, compressed.length); + System.arraycopy(compressed, 0, block, BLOCK_HEADER_LENGTH, compressed.length); + + byte[] frame = new byte[CHECKSUM_LENGTH + block.length]; + System.arraycopy(block, 0, frame, CHECKSUM_LENGTH, block.length); + reseal(frame); + return frame; + } - long[] checksum = ClickHouseCityHash.cityHash128(block, 0, block.length); - byte[] frame = new byte[16 + block.length]; + /** + * Recomputes the checksum of a frame, so a block mutated after {@link #frame(byte, byte[])} + * still passes the checksum and reaches the decompression of the reader. The checksum covers + * the compressed size the header declares, which is what the reader hashes. + */ + private static void reseal(byte[] frame) { + long[] checksum = ClickHouseCityHash.cityHash128(frame, CHECKSUM_LENGTH, + ClickHouseLZ4InputStream.getInt32(frame, COMPRESSED_SIZE_OFFSET)); setInt64(frame, 0, checksum[0]); setInt64(frame, 8, checksum[1]); - System.arraycopy(block, 0, frame, 16, block.length); - return frame; } private static void setInt64(byte[] bytes, int offset, long value) { From fe6d6133e34d795a34bc4e187a64090ac4485626 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:47:25 +0000 Subject: [PATCH 3/8] Request a compressed response with the algorithm the client decodes The response of a query was requested with the compress=1 framing of the HTTP interface, whose codec the server chooses on its own. ClickHouse 26.9 changed that codec from LZ4 to ZSTD(3) and the framed output follows the built-in default with no setting to override it, so every compressed read failed with 'Invalid LZ4 magic byte'. The algorithm is now part of the request: the new client.compression_algorithm property names it out of a defined set, defaults to LZ4 for backward compatibility, and is sent as the content coding of the operation, so a compressed body always uses the algorithm the client asked for. Fixes: https://github.com/ClickHouse/clickhouse-java/issues/3105 --- CHANGELOG.md | 25 ++- clickhouse-jdbc/pom.xml | 3 +- client-v2/pom.xml | 13 +- .../com/clickhouse/client/api/Client.java | 29 ++- .../client/api/ClientConfigProperties.java | 21 ++ .../api/enums/CompressionAlgorithm.java | 71 ++++++ .../internal/ClickHouseLZ4InputStream.java | 71 +----- .../api/internal/HttpAPIClientHelper.java | 62 ++--- .../api/ClientConfigPropertiesTest.java | 41 +++- .../ClickHouseLZ4InputStreamTest.java | 212 ------------------ .../QueryServerContentCompressionTests.java | 48 +++- docs/features.md | 2 +- docs/releases/0_11_0.md | 31 +++ 13 files changed, 293 insertions(+), 336 deletions(-) create mode 100644 client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java delete mode 100644 client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index b99ef3c29..18b11238d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ ### Breaking Changes +- **[client-v2] The algorithm of a compressed response is now requested with `Accept-Encoding` and is no longer the + one the server picks.** A response was requested with the `compress=1` framing, whose codec the server chooses on + its own: ClickHouse `26.9` changed that codec from `LZ4` to `ZSTD(3)`, the framed output follows the built-in + default and no setting overrides it, so the client could not keep reading a response it asked for. A response is + now requested with the content coding of the new `client.compression_algorithm` property + (`Client.Builder#compressionAlgorithm`), which defaults to `LZ4` and keeps the algorithm of a compressed body the + same on every server version. Set the property to `ZSTD`, `GZIP` or `NONE` to select another algorithm; `ZSTD` + needs `com.github.luben:zstd-jni` on the classpath, which stays a `provided` dependency. A client that reads a + compressed response now also sends `enable_http_compression=1`, which a user profile that forbids setting changes + (`readonly = 1`) rejects - such a profile has to use `readonly = 2` or `client.compression_algorithm = NONE`. + (https://github.com/ClickHouse/clickhouse-java/issues/3105) + - **[client-v2]** `com.clickhouse.client.api.metrics.OperationMetrics` now has a single constructor, `OperationMetrics(ClientStatisticsHolder, OperationType)`; the constructor without an operation type was removed. Metrics are created by the client, which always knows the kind of the operation it runs, and the constructor takes @@ -138,13 +150,12 @@ ### Bug Fixes - **[client-v2]** Fixed every compressed read failing with `Invalid LZ4 magic byte: '-112'` against ClickHouse `26.9` - and later. The server picks the codec of the `compress=1` framing and now uses `ZSTD(3)` by default, while the - response reader asserted the LZ4 method byte of every block, so any query answered with a compressed body died - before the first row was parsed and no setting could restore the old codec. The reader now takes the codec from the - block header, which is what makes the framing self-describing, and decompresses LZ4, ZSTD and uncompressed blocks — - including a body that mixes them. A block of an unknown codec is still rejected, so a response that is not framed at - all is reported as before. `zstd-jni` became a required dependency of `client-v2` and is shaded into the `all` - artifacts. (https://github.com/ClickHouse/clickhouse-java/issues/3105) + and later. The server chooses the codec of the `compress=1` framing the client requested and switched that codec to + `ZSTD(3)`, while the response reader asserted the LZ4 method byte of every block, so any query answered with a + compressed body died before the first row was parsed. The algorithm of a response is now requested with + `Accept-Encoding`, so the client reads the algorithm it asked for; see the breaking-changes entry above. + (https://github.com/ClickHouse/clickhouse-java/issues/3105) + - **[jdbc-v2]** Fixed `DatabaseMetaData#getTables` reporting `TABLE_TYPE = TABLE` for a table with the `BigQuery` engine (present in `system.table_engines` since ClickHouse `26.8`). The engine was missing from the engine-to-table-type mapping, so it fell back to the default `TABLE`, and `getTables(..., types = {"REMOTE TABLE"})` diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml index 495db0ade..c4c56727f 100644 --- a/clickhouse-jdbc/pom.xml +++ b/clickhouse-jdbc/pom.xml @@ -58,11 +58,10 @@ lz4-java ${lz4.version} - com.github.luben zstd-jni + provided org.slf4j diff --git a/client-v2/pom.xml b/client-v2/pom.xml index 40869a710..c1b4e92d2 100644 --- a/client-v2/pom.xml +++ b/client-v2/pom.xml @@ -58,13 +58,6 @@ ${lz4.version} - - - com.github.luben - zstd-jni - - org.apache.commons commons-compress @@ -195,6 +188,12 @@ 5.19.0 test + + com.github.luben + zstd-jni + 1.5.7-6 + test + org.bouncycastle bcprov-jdk18on diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 786fbfe5e..f6762f3d6 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -11,6 +11,7 @@ import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader; import com.clickhouse.client.api.data_formats.internal.MapBackedRecord; import com.clickhouse.client.api.data_formats.internal.ProcessParser; +import com.clickhouse.client.api.enums.CompressionAlgorithm; import com.clickhouse.client.api.enums.Protocol; import com.clickhouse.client.api.enums.ProxyType; import com.clickhouse.client.api.enums.SSLMode; @@ -656,10 +657,32 @@ public Builder compressClientRequest(boolean enabled) { return this; } + /** + * Algorithm of a compressed request or response body. The algorithm is requested with the HTTP + * content-coding of the operation, so a compressed body always uses the algorithm set here and + * never one the server picks on its own. {@link CompressionAlgorithm#NONE} disables compression. + * Default is {@link CompressionAlgorithm#LZ4}. + *

+ * {@link CompressionAlgorithm#ZSTD} needs {@code com.github.luben:zstd-jni} on the classpath - the + * dependency is {@code provided}, so an application that selects the algorithm has to declare it. + *

+ * A request body follows this algorithm only together with {@link #useHttpCompression(boolean)}; + * the ClickHouse framing of a request compressed without it is always LZ4. + * + * @param algorithm - algorithm of a compressed body + * @return same instance of the builder + */ + public Builder compressionAlgorithm(CompressionAlgorithm algorithm) { + ValidationUtils.checkNotNull(algorithm, "algorithm"); + this.configuration.put(ClientConfigProperties.COMPRESSION_ALGORITHM.getKey(), algorithm.name()); + return this; + } + /** * Configures the client to use HTTP compression. In this case compression is controlled by - * http headers. Client compression will set {@code Content-Encoding: lz4} header and server - * compression will set {@code Accept-Encoding: lz4} header. Default is false. + * http headers. Client compression will set the {@code Content-Encoding} header and server + * compression will set the {@code Accept-Encoding} header, both to the content coding of + * {@link #compressionAlgorithm(CompressionAlgorithm)}. Default is false. * * @param enabled - indicates if http compression is enabled * @return @@ -695,8 +718,6 @@ public Builder setLZ4UncompressedBufferSize(int size) { /** * Disable native compression. If set to true then native compression will be disabled. * If from some reason the native compressor is not working then it can be disabled. - * Applies to LZ4 only: a response the server compressed with ZSTD is always read with the - * native library of zstd-jni, because the server picks the codec of the response. * @param disable * @return */ diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index bbce7a37b..b2dfa8c46 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -2,6 +2,7 @@ import com.clickhouse.client.api.data_formats.ClickHouseFormatReader; import com.clickhouse.client.api.data_formats.internal.AbstractBinaryFormatReader; +import com.clickhouse.client.api.enums.CompressionAlgorithm; import com.clickhouse.client.api.enums.SSLMode; import com.clickhouse.client.api.internal.ClickHouseLZ4OutputStream; import com.clickhouse.data.ClickHouseDataType; @@ -245,6 +246,26 @@ public Object parseValue(String value) { .collect(Collectors.toList()); } }, + + /** + * Algorithm of a compressed request or response body. The algorithm is requested with the HTTP + * content-coding of the operation ({@code Accept-Encoding} for a response, {@code Content-Encoding} + * for a request), so a compressed body always uses the algorithm the client asked for and never one + * the server picks on its own. {@link CompressionAlgorithm#NONE} disables compression of both + * directions. + *

+ * The name of an algorithm and its content-coding token are both accepted, in any case. + *

+ * Appended at the end of the enum on purpose: adding a constant in the middle would shift the ordinal + * of every following constant (see {@code docs/changes_checklist.md}). + */ + COMPRESSION_ALGORITHM("client.compression_algorithm", CompressionAlgorithm.class, + CompressionAlgorithm.LZ4.name()) { + @Override + public Object parseValue(String value) { + return value == null ? null : CompressionAlgorithm.fromValue(value); + } + }, ; private static final Logger LOG = LoggerFactory.getLogger(ClientConfigProperties.class); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java b/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java new file mode 100644 index 000000000..9b79162a8 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java @@ -0,0 +1,71 @@ +package com.clickhouse.client.api.enums; + +/** + * Enumerates the compression algorithms the client can ask the server for and can apply itself. + * + *

The algorithm is requested with the HTTP content-coding of the operation - {@code Accept-Encoding} + * for a response and {@code Content-Encoding} for a request - so a compressed body always uses the + * algorithm of the request and never one the server picks on its own.

+ * + * + */ +public enum CompressionAlgorithm { + + /** + * ClickHouse LZ4. Default algorithm. + */ + LZ4("lz4"), + + /** + * Zstandard. Requires {@code com.github.luben:zstd-jni} on the classpath. + */ + ZSTD("zstd"), + + /** + * gzip. Supported by the JDK. + */ + GZIP("gzip"), + + /** + * No compression. + */ + NONE("none"); + + private final String httpContentCoding; + + CompressionAlgorithm(String httpContentCoding) { + this.httpContentCoding = httpContentCoding; + } + + /** + * Returns the HTTP content-coding token of the algorithm, as used in the {@code Accept-Encoding} + * and {@code Content-Encoding} headers. + * + * @return content-coding token + */ + public String getHttpContentCoding() { + return httpContentCoding; + } + + /** + * Case-insensitive variant of {@link #valueOf(String)} that also accepts the content-coding token. + * + * @param value algorithm name or content-coding token in any case + * @return matching algorithm + * @throws IllegalArgumentException when the value does not match any algorithm + */ + public static CompressionAlgorithm fromValue(String value) { + for (CompressionAlgorithm algorithm : values()) { + if (algorithm.name().equalsIgnoreCase(value) || algorithm.httpContentCoding.equalsIgnoreCase(value)) { + return algorithm; + } + } + throw new IllegalArgumentException("Unknown compression algorithm '" + value + "'"); + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java index aff39fcee..8550fb6c2 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStream.java @@ -4,8 +4,6 @@ import com.clickhouse.data.ClickHouseByteUtils; import com.clickhouse.data.ClickHouseCityHash; import com.clickhouse.data.ClickHouseUtils; -import com.github.luben.zstd.Zstd; -import com.github.luben.zstd.ZstdException; import net.jpountz.lz4.LZ4FastDecompressor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,11 +13,6 @@ import java.io.InputStream; import java.nio.ByteBuffer; -/** - * Reads the framed output of the ClickHouse HTTP {@code compress=1} interface. Each block is - * self-describing: its header carries the compression method the server used, so the codec is - * detected per block instead of being assumed. LZ4, ZSTD and uncompressed blocks are supported. - */ public class ClickHouseLZ4InputStream extends InputStream { private static Logger LOG = LoggerFactory.getLogger(ClickHouseLZ4InputStream.class); @@ -34,7 +27,7 @@ public class ClickHouseLZ4InputStream extends InputStream { public ClickHouseLZ4InputStream(InputStream in, LZ4FastDecompressor decompressor, int bufferSize) { super(); - LOG.debug("Reading compressed response with buffer size {}", bufferSize); + LOG.debug("Using LZ4 decompressor with buffer size {}", bufferSize); this.decompressor = decompressor; this.in = in; this.buffer = ByteBuffer.allocate(bufferSize); @@ -73,8 +66,6 @@ public int read(byte[] b, int off, int len) throws IOException { static final byte MAGIC = (byte) 0x82; - static final byte MAGIC_ZSTD = (byte) 0x90; - static final byte MAGIC_NONE = (byte) 0x02; static final int HEADER_LENGTH = 25; final byte[] headerBuff = new byte[HEADER_LENGTH]; @@ -116,10 +107,9 @@ private int refill() throws IOException { return -1; } - // 1 byte - compression method (0x82 LZ4, 0x90 ZSTD, 0x02 uncompressed) - final byte method = headerBuff[16]; - if (method != MAGIC && method != MAGIC_ZSTD && method != MAGIC_NONE) { - throw new ClientException("Invalid compression method byte: '" + method + "'"); + if (headerBuff[16] != MAGIC) { + // 1 byte - 0x82 (shows this is LZ4) + throw new ClientException("Invalid LZ4 magic byte: '" + headerBuff[16] + "'"); } // 4 bytes - size of the compressed data including 9 bytes of the header @@ -128,14 +118,8 @@ private int refill() throws IOException { int uncompressedSize = getInt32(headerBuff, 21); int offset = 9; - if (compressedSizeWithHeader < offset || uncompressedSize < 0) { - throw new ClientException(ClickHouseUtils.format( - "Corrupted stream: block declares %s compressed and %s uncompressed bytes", - compressedSizeWithHeader, uncompressedSize)); - } - final byte[] block = new byte[compressedSizeWithHeader]; - block[0] = method; + block[0] = MAGIC; setInt32(block, 1, compressedSizeWithHeader); setInt32(block, 5, uncompressedSize); // compressed data: compressed_size - 9 bytes @@ -154,55 +138,12 @@ private int refill() throws IOException { if (buffer.capacity() < uncompressedSize) { buffer = ByteBuffer.allocate(uncompressedSize); } - decompress(method, block, offset, remaining, uncompressedSize); + decompressor.decompress(ByteBuffer.wrap(block), offset, buffer, 0, uncompressedSize); buffer.position(0); buffer.limit(uncompressedSize); return uncompressedSize; } - /** - * Decompresses a single block into {@link #buffer} with the codec the block header declares. - * - * @param method compression method byte of the block - * @param block block, including its 9 bytes of header - * @param offset offset of the compressed data in the block - * @param compressedSize size of the compressed data - * @param uncompressedSize size of the data after decompression - */ - private void decompress(byte method, byte[] block, int offset, int compressedSize, int uncompressedSize) { - switch (method) { - case MAGIC: - decompressor.decompress(ByteBuffer.wrap(block), offset, buffer, 0, uncompressedSize); - break; - case MAGIC_ZSTD: - long decompressedSize; - try { - decompressedSize = Zstd.decompressByteArray(buffer.array(), buffer.arrayOffset(), - uncompressedSize, block, offset, compressedSize); - } catch (ZstdException e) { - throw new ClientException("Failed to decompress ZSTD block: " + e.getMessage(), e); - } catch (LinkageError e) { - // the server picks the codec of the response, so ZSTD cannot be avoided by configuration - throw new ClientException("Server compressed the response with ZSTD but the native library of " - + "zstd-jni is not available on this platform", e); - } - if (decompressedSize != uncompressedSize) { - throw new ClientException(ClickHouseUtils.format( - "Corrupted stream: decompressed %s bytes while %s were expected", - decompressedSize, uncompressedSize)); - } - break; - default: // MAGIC_NONE - if (compressedSize != uncompressedSize) { - throw new ClientException(ClickHouseUtils.format( - "Corrupted stream: uncompressed block holds %s bytes while %s were expected", - compressedSize, uncompressedSize)); - } - System.arraycopy(block, offset, buffer.array(), buffer.arrayOffset(), uncompressedSize); - break; - } - } - /** * Read int32 Little Endian * @param bytes diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 9b303a960..e944f9424 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -11,6 +11,7 @@ import com.clickhouse.client.api.DataTransferException; import com.clickhouse.client.api.ServerException; import com.clickhouse.client.api.TransportException; +import com.clickhouse.client.api.enums.CompressionAlgorithm; import com.clickhouse.client.api.enums.ProxyType; import com.clickhouse.client.api.enums.SSLMode; import com.clickhouse.client.api.http.ClickHouseHttpProto; @@ -123,7 +124,6 @@ public class HttpAPIClientHelper { private static final int ERROR_BODY_BUFFER_SIZE = 1024; // Error messages are usually small - private final String DEFAULT_HTTP_COMPRESSION_ALGO = "lz4"; private static final Pattern PATTERN_HEADER_VALUE_ASCII = Pattern.compile( "\\p{Graph}+(?:[ ]\\p{Graph}+)*"); @@ -447,7 +447,7 @@ private ServerException readNotClickHouseError(HttpEntity httpEntity, String que } break; } catch (ClientException e) { - // response body is not the framed output of the compress=1 interface + // Invalid LZ4 Magic if (body instanceof ClickHouseLZ4InputStream) { ClickHouseLZ4InputStream stream = (ClickHouseLZ4InputStream) body; body = stream.getInputStream(); @@ -478,7 +478,7 @@ private static ServerException readClickHouseError(HttpEntity httpEntity, int se try { rBytes = body.read(buffer); } catch (ClientException e) { - // response body is not the framed output of the compress=1 interface + // Invalid LZ4 Magic if (body instanceof ClickHouseLZ4InputStream) { ClickHouseLZ4InputStream stream = (ClickHouseLZ4InputStream) body; body = stream.getInputStream(); @@ -925,14 +925,17 @@ private void addHeaders(HttpPost req, Map requestConfig) { boolean serverCompression = ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getOrDefault(requestConfig); boolean useHttpCompression = ClientConfigProperties.USE_HTTP_COMPRESSION.getOrDefault(requestConfig); boolean appCompressedData = ClientConfigProperties.APP_COMPRESSED_DATA.getOrDefault(requestConfig); + CompressionAlgorithm algorithm = ClientConfigProperties.COMPRESSION_ALGORITHM.getOrDefault(requestConfig); - if (useHttpCompression) { + if (algorithm != CompressionAlgorithm.NONE) { if (serverCompression) { - setHeader(req, HttpHeaders.ACCEPT_ENCODING, DEFAULT_HTTP_COMPRESSION_ALGO); + // the codec of a compressed response is the one requested here: the server picks its own + // default codec for the compress=1 framing and does not let a client select it + setHeader(req, HttpHeaders.ACCEPT_ENCODING, algorithm.getHttpContentCoding()); } - if (clientCompression && !appCompressedData) { - setHeader(req, HttpHeaders.CONTENT_ENCODING, DEFAULT_HTTP_COMPRESSION_ALGO); + if (useHttpCompression && clientCompression && !appCompressedData) { + setHeader(req, HttpHeaders.CONTENT_ENCODING, algorithm.getHttpContentCoding()); } } @@ -971,17 +974,24 @@ private void addRequestParams(Map requestConfig, BiConsumer boolean clientCompression = ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getOrDefault(requestConfig); boolean useHttpCompression = ClientConfigProperties.USE_HTTP_COMPRESSION.getOrDefault(requestConfig); boolean appCompressedData = ClientConfigProperties.APP_COMPRESSED_DATA.getOrDefault(requestConfig); + CompressionAlgorithm algorithm = ClientConfigProperties.COMPRESSION_ALGORITHM.getOrDefault(requestConfig); if (httpEntity.getContentEncoding() != null && !appCompressedData) { // http header is set and data is not compressed return new CompressedEntity(httpEntity, false, CompressorStreamFactory.getSingleton()); - } else if (clientCompression && !appCompressedData) { + } else if (clientCompression && !appCompressedData && algorithm != CompressionAlgorithm.NONE) { int buffSize = ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getOrDefault(requestConfig); return new LZ4Entity(httpEntity, useHttpCompression, false, true, buffSize, false, lz4Factory); @@ -1026,20 +1037,13 @@ private HttpEntity wrapRequestEntity(HttpEntity httpEntity, Map } private HttpEntity wrapResponseEntity(HttpEntity httpEntity, int httpStatus, Map requestConfig) { - boolean serverCompression = ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getOrDefault(requestConfig); - boolean useHttpCompression = ClientConfigProperties.USE_HTTP_COMPRESSION.getOrDefault(requestConfig); - if (httpEntity.getContentEncoding() != null) { - // http compressed response + // the algorithm of a compressed response is the one the request asked for return new CompressedEntity(httpEntity, true, CompressorStreamFactory.getSingleton()); } - // data compression - if (serverCompression && !(httpStatus == HttpStatus.SC_FORBIDDEN || httpStatus == HttpStatus.SC_UNAUTHORIZED)) { - int buffSize = ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getOrDefault(requestConfig); - return new LZ4Entity(httpEntity, useHttpCompression, true, false, buffSize, true, lz4Factory); - } - + // a response without a content coding is not compressed: the server answers an unsupported + // Accept-Encoding, and a request that asks for no compression, with a plain body return httpEntity; } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java index 9c23ca522..43991c051 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java @@ -1,6 +1,7 @@ package com.clickhouse.client.api; +import com.clickhouse.client.api.enums.CompressionAlgorithm; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -76,4 +77,42 @@ public void testParseConfigMapSanitizesSslCipherSuites() { Assert.assertEquals(parsed.get(ClientConfigProperties.SSL_CIPHER_SUITES.getKey()), Arrays.asList("TLS_AES_256_GCM_SHA384", "TLS_AES_128_GCM_SHA256")); } -} \ No newline at end of file + + @DataProvider(name = "compressionAlgorithms") + public static Object[][] compressionAlgorithms() { + return new Object[][]{ + // raw client.compression_algorithm value -> expected algorithm + {"LZ4", CompressionAlgorithm.LZ4}, + {"lz4", CompressionAlgorithm.LZ4}, + {"ZSTD", CompressionAlgorithm.ZSTD}, + {"zstd", CompressionAlgorithm.ZSTD}, + {"GZIP", CompressionAlgorithm.GZIP}, + {"gzip", CompressionAlgorithm.GZIP}, + {"NONE", CompressionAlgorithm.NONE}, + {"none", CompressionAlgorithm.NONE}, + }; + } + + @Test(groups = {"unit"}, dataProvider = "compressionAlgorithms") + public void testCompressionAlgorithmParsed(String raw, CompressionAlgorithm expected) { + Assert.assertEquals(ClientConfigProperties.COMPRESSION_ALGORITHM.parseValue(raw), expected); + + Map config = new HashMap<>(); + config.put(ClientConfigProperties.COMPRESSION_ALGORITHM.getKey(), raw); + Assert.assertEquals(ClientConfigProperties.parseConfigMap(config) + .get(ClientConfigProperties.COMPRESSION_ALGORITHM.getKey()), expected); + } + + @Test(groups = {"unit"}) + public void testCompressionAlgorithmDefaultsToLz4() { + Assert.assertEquals(ClientConfigProperties.COMPRESSION_ALGORITHM.getDefObjVal(), CompressionAlgorithm.LZ4); + Assert.assertEquals( + ClientConfigProperties.COMPRESSION_ALGORITHM.getOrDefault(Collections.emptyMap()), + CompressionAlgorithm.LZ4); + } + + @Test(groups = {"unit"}, expectedExceptions = IllegalArgumentException.class) + public void testUnknownCompressionAlgorithmRejected() { + ClientConfigProperties.COMPRESSION_ALGORITHM.parseValue("snappy"); + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java deleted file mode 100644 index 5283d4ad8..000000000 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/ClickHouseLZ4InputStreamTest.java +++ /dev/null @@ -1,212 +0,0 @@ -package com.clickhouse.client.api.internal; - -import com.clickhouse.client.api.ClientException; -import com.clickhouse.data.ClickHouseCityHash; -import com.github.luben.zstd.Zstd; -import com.github.luben.zstd.ZstdException; -import net.jpountz.lz4.LZ4Factory; -import org.testng.Assert; -import org.testng.annotations.DataProvider; -import org.testng.annotations.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; - -public class ClickHouseLZ4InputStreamTest { - - private static final int BUFFER_SIZE = 8192; - - private static final int CHECKSUM_LENGTH = 16; - - private static final int BLOCK_HEADER_LENGTH = 9; - - private static final int METHOD_OFFSET = CHECKSUM_LENGTH; - - private static final int COMPRESSED_SIZE_OFFSET = CHECKSUM_LENGTH + 1; - - private static final int UNCOMPRESSED_SIZE_OFFSET = CHECKSUM_LENGTH + 5; - - private static final int DATA_OFFSET = CHECKSUM_LENGTH + BLOCK_HEADER_LENGTH; - - @Test(groups = {"unit"}, dataProvider = "compressionMethodProvider") - public void testReadsBlockOfEveryCompressionMethod(byte method) throws IOException { - byte[] payload = payload(); - - Assert.assertEquals(readFully(frame(method, payload), BUFFER_SIZE), payload); - } - - @Test(groups = {"unit"}, dataProvider = "compressionMethodProvider") - public void testReadsBlockLargerThanInitialBuffer(byte method) throws IOException { - byte[] payload = payload(); - - Assert.assertEquals(readFully(frame(method, payload), 16), payload); - } - - @DataProvider(name = "compressionMethodProvider") - public Object[][] compressionMethodProvider() { - return new Object[][]{ - {ClickHouseLZ4InputStream.MAGIC}, - {ClickHouseLZ4InputStream.MAGIC_ZSTD}, - {ClickHouseLZ4InputStream.MAGIC_NONE}, - }; - } - - @Test(groups = {"unit"}) - public void testReadsConsecutiveBlocksOfDifferentMethods() throws IOException { - byte[] first = "first block\n".getBytes(StandardCharsets.UTF_8); - byte[] second = "second block\n".getBytes(StandardCharsets.UTF_8); - ByteArrayOutputStream stream = new ByteArrayOutputStream(); - stream.write(frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, first)); - stream.write(frame(ClickHouseLZ4InputStream.MAGIC, second)); - - byte[] expected = new byte[first.length + second.length]; - System.arraycopy(first, 0, expected, 0, first.length); - System.arraycopy(second, 0, expected, first.length, second.length); - Assert.assertEquals(readFully(stream.toByteArray(), BUFFER_SIZE), expected); - } - - @Test(groups = {"unit"}, dataProvider = "corruptedFrameProvider") - public void testRejectsCorruptedFrame(byte[] frame, String expectedMessage, Class expectedCause) { - ClientException e = Assert.expectThrows(ClientException.class, () -> readFully(frame, BUFFER_SIZE)); - Assert.assertTrue(e.getMessage().contains(expectedMessage), e.getMessage()); - if (expectedCause == null) { - Assert.assertNull(e.getCause()); - } else { - Assert.assertTrue(expectedCause.isInstance(e.getCause()), String.valueOf(e.getCause())); - } - } - - @DataProvider(name = "corruptedFrameProvider") - public Object[][] corruptedFrameProvider() { - byte[] payload = payload(); - - byte[] unknownMethod = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); - unknownMethod[METHOD_OFFSET] = (byte) 0x42; - - byte[] impossibleCompressedSize = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); - ClickHouseLZ4InputStream.setInt32(impossibleCompressedSize, COMPRESSED_SIZE_OFFSET, 4); - - byte[] negativeUncompressedSize = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); - ClickHouseLZ4InputStream.setInt32(negativeUncompressedSize, UNCOMPRESSED_SIZE_OFFSET, -1); - - byte[] corruptedPayload = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); - corruptedPayload[corruptedPayload.length - 1] ^= 0xFF; - - // the ZSTD frame of the block loses its magic, so the codec itself rejects the data - byte[] undecompressibleZstdBlock = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); - Arrays.fill(undecompressibleZstdBlock, DATA_OFFSET, DATA_OFFSET + 4, (byte) 0); - reseal(undecompressibleZstdBlock); - - byte[] shortZstdBlock = frame(ClickHouseLZ4InputStream.MAGIC_ZSTD, payload); - ClickHouseLZ4InputStream.setInt32(shortZstdBlock, UNCOMPRESSED_SIZE_OFFSET, payload.length + 8); - reseal(shortZstdBlock); - - byte[] shortUncompressedBlock = frame(ClickHouseLZ4InputStream.MAGIC_NONE, payload); - ClickHouseLZ4InputStream.setInt32(shortUncompressedBlock, UNCOMPRESSED_SIZE_OFFSET, payload.length - 1); - reseal(shortUncompressedBlock); - - return new Object[][]{ - {unknownMethod, "Invalid compression method byte", null}, - {impossibleCompressedSize, "block declares 4 compressed", null}, - {negativeUncompressedSize, "-1 uncompressed bytes", null}, - {corruptedPayload, "checksum mismatch", null}, - {undecompressibleZstdBlock, "Failed to decompress ZSTD block", ZstdException.class}, - {shortZstdBlock, "decompressed " + payload.length + " bytes while " - + (payload.length + 8) + " were expected", null}, - {shortUncompressedBlock, "uncompressed block holds " + payload.length + " bytes while " - + (payload.length - 1) + " were expected", null}, - }; - } - - @Test(groups = {"unit"}) - public void testKeepsUnframedBodyReadableThroughHeaderBuffer() throws IOException { - byte[] body = "Code: 62. DB::Exception: Syntax error: failed at position 1\n" - .getBytes(StandardCharsets.UTF_8); - - try (ClickHouseLZ4InputStream in = new ClickHouseLZ4InputStream(new ByteArrayInputStream(body), - LZ4Factory.fastestInstance().fastDecompressor(), BUFFER_SIZE)) { - Assert.expectThrows(ClientException.class, () -> in.read(new byte[64])); - Assert.assertEquals(in.getHeaderBuffer(), - Arrays.copyOf(body, ClickHouseLZ4InputStream.HEADER_LENGTH)); - } - } - - private static byte[] payload() { - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < 100; i++) { - sb.append(i).append('\t').append("value-").append(i).append('\n'); - } - return sb.toString().getBytes(StandardCharsets.UTF_8); - } - - private static byte[] readFully(byte[] frames, int bufferSize) throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - try (InputStream in = new ClickHouseLZ4InputStream(new ByteArrayInputStream(frames), - LZ4Factory.fastestInstance().fastDecompressor(), bufferSize)) { - byte[] chunk = new byte[64]; - int read; - while ((read = in.read(chunk)) != -1) { - out.write(chunk, 0, read); - } - } - return out.toByteArray(); - } - - /** - * Builds a block the way the server frames the output of the HTTP {@code compress=1} - * interface: 16 bytes of CityHash128 checksum over the block, then a 9 byte header holding the - * compression method, the compressed size (header included) and the uncompressed size, then - * the compressed data. - */ - private static byte[] frame(byte method, byte[] data) { - byte[] compressed; - switch (method) { - case ClickHouseLZ4InputStream.MAGIC: - byte[] lz4 = new byte[LZ4Factory.fastestInstance().fastCompressor().maxCompressedLength(data.length)]; - int lz4Length = LZ4Factory.fastestInstance().fastCompressor() - .compress(data, 0, data.length, lz4, 0, lz4.length); - compressed = new byte[lz4Length]; - System.arraycopy(lz4, 0, compressed, 0, lz4Length); - break; - case ClickHouseLZ4InputStream.MAGIC_ZSTD: - compressed = Zstd.compress(data, 3); - break; - default: - compressed = data; - break; - } - - byte[] block = new byte[BLOCK_HEADER_LENGTH + compressed.length]; - block[0] = method; - ClickHouseLZ4InputStream.setInt32(block, 1, block.length); - ClickHouseLZ4InputStream.setInt32(block, 5, data.length); - System.arraycopy(compressed, 0, block, BLOCK_HEADER_LENGTH, compressed.length); - - byte[] frame = new byte[CHECKSUM_LENGTH + block.length]; - System.arraycopy(block, 0, frame, CHECKSUM_LENGTH, block.length); - reseal(frame); - return frame; - } - - /** - * Recomputes the checksum of a frame, so a block mutated after {@link #frame(byte, byte[])} - * still passes the checksum and reaches the decompression of the reader. The checksum covers - * the compressed size the header declares, which is what the reader hashes. - */ - private static void reseal(byte[] frame) { - long[] checksum = ClickHouseCityHash.cityHash128(frame, CHECKSUM_LENGTH, - ClickHouseLZ4InputStream.getInt32(frame, COMPRESSED_SIZE_OFFSET)); - setInt64(frame, 0, checksum[0]); - setInt64(frame, 8, checksum[1]); - } - - private static void setInt64(byte[] bytes, int offset, long value) { - for (int i = 0; i < 8; i++) { - bytes[offset + i] = (byte) (0xFF & (value >> (8 * i))); - } - } -} diff --git a/client-v2/src/test/java/com/clickhouse/client/query/QueryServerContentCompressionTests.java b/client-v2/src/test/java/com/clickhouse/client/query/QueryServerContentCompressionTests.java index 1001b08be..9dc385318 100644 --- a/client-v2/src/test/java/com/clickhouse/client/query/QueryServerContentCompressionTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/query/QueryServerContentCompressionTests.java @@ -1,8 +1,40 @@ -package com.clickhouse.client.query; - -public class QueryServerContentCompressionTests extends QueryTests { - - QueryServerContentCompressionTests() { - super(true, false); - } -} +package com.clickhouse.client.query; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.enums.CompressionAlgorithm; +import com.clickhouse.client.api.query.GenericRecord; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.List; + +public class QueryServerContentCompressionTests extends QueryTests { + + QueryServerContentCompressionTests() { + super(true, false); + } + + @Test(groups = {"integration"}, dataProvider = "compressionAlgorithms") + public void testQueryWithCompressionAlgorithm(CompressionAlgorithm algorithm) throws Exception { + try (Client client = newClient().compressionAlgorithm(algorithm).build()) { + List records = client.queryAll("SELECT number, toString(number) AS str " + + "FROM system.numbers LIMIT 1000"); + + Assert.assertEquals(records.size(), 1000); + Assert.assertEquals(records.get(0).getLong("number"), 0); + Assert.assertEquals(records.get(999).getLong("number"), 999); + Assert.assertEquals(records.get(999).getString("str"), "999"); + } + } + + @DataProvider(name = "compressionAlgorithms") + public Object[][] compressionAlgorithms() { + return new Object[][]{ + {CompressionAlgorithm.LZ4}, + {CompressionAlgorithm.ZSTD}, + {CompressionAlgorithm.GZIP}, + {CompressionAlgorithm.NONE}, + }; + } +} diff --git a/docs/features.md b/docs/features.md index 8df78cace..3647168b0 100644 --- a/docs/features.md +++ b/docs/features.md @@ -31,7 +31,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Session handling: Supports client-wide and per-operation HTTP sessions, operation-level session overrides, runtime updates of client `session_id`, and server-side session validation through `session_check`. - Metadata discovery: Loads table schemas from table names or queries and allows schema registration for typed read/write operations. - Server information loading: Can refresh server version, current user, and server time zone information. -- Compression support: Supports response compression, ClickHouse LZ4 request compression, HTTP content compression, and caller-supplied precompressed insert bodies. A compressed response is read with the codec of each block, so the server is free to answer with LZ4, ZSTD or uncompressed blocks. +- Compression support: Supports response compression, ClickHouse LZ4 request compression, HTTP content compression, and caller-supplied precompressed insert bodies. The algorithm of a compressed body is selected with `client.compression_algorithm` (`LZ4` by default, also `ZSTD`, `GZIP` and `NONE`) and is requested with the HTTP content coding of the operation, so a compressed response uses the algorithm the client asked for on every server version. - Retry behavior: Can retry failed operations for configured failure causes and retry limits. - Client-side request cancellation: `Client.cancelTransportRequest(String queryId)` aborts the in-flight HTTP request and its IO for the operation started with the given query id. It requires the caller to set the query id in operation settings, is best-effort (it cancels client-side IO but the result is not guaranteed), and does not issue a server-side `KILL QUERY` - the server stops the query on its own once the client disconnects. A cancelled operation that is being retried stops instead of issuing another request, also when the cancellation lands between two attempts (for example from `DataStreamWriter#onRetry()`). - Metrics and observability: Exposes client/server operation metrics and optionally integrates connection-pool gauges with Micrometer. diff --git a/docs/releases/0_11_0.md b/docs/releases/0_11_0.md index 6c40120f9..682e90089 100644 --- a/docs/releases/0_11_0.md +++ b/docs/releases/0_11_0.md @@ -2,6 +2,37 @@ # Migration Guide +## CLIENT-V2: A Compressed Response Uses the Algorithm the Client Asks For + +A compressed response was requested with the `compress=1` framing of the HTTP interface, whose codec the server +chooses on its own. ClickHouse `26.9` changed that codec from `LZ4` to `ZSTD(3)`; the framed output follows the +built-in default codec and no setting overrides it, so a client that expects `LZ4` cannot read the response of a +`26.9` server at all. + +The client now requests a response with the HTTP content coding of the algorithm it will decode, so the algorithm of +a compressed body is always the one the client asked for: + +- The new property `client.compression_algorithm` (builder method `Client.Builder#compressionAlgorithm`) selects the + algorithm out of `LZ4` (default), `ZSTD`, `GZIP` and `NONE`. Both the name and the content-coding token of an + algorithm are accepted, in any case. +- The default is `LZ4`, so an application that does not set the property keeps reading `LZ4` on every server version. +- `ZSTD` needs `com.github.luben:zstd-jni` on the classpath. The dependency stays `provided`, so an application that + selects `ZSTD` has to declare it. +- `NONE` disables compression of the request and the response, whatever `compress` and `decompress` are set to. + +What to check in an application: + +- **A user profile that forbids setting changes.** A client that reads a compressed response now also sends + `enable_http_compression=1`, which the server rejects for a profile with `readonly = 1`. Use `readonly = 2`, which + allows setting changes, or set `client.compression_algorithm` to `NONE`. +- **Code that inspects the response encoding.** A compressed response now carries `Content-Encoding` with the + requested coding instead of the `compress=1` framing of ClickHouse. + +The algorithm of a compressed *request* body is unchanged: it follows `client.compression_algorithm` only together +with `Client.Builder#useHttpCompression`, and the ClickHouse framing of a request compressed without it is always +`LZ4`. + + ## CLIENT-V2: `OperationMetrics` Has a Single Constructor `com.clickhouse.client.api.metrics.OperationMetrics` now has one constructor, From c8fd801e440b36226341ca45f7efa92a4458d6d0 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:01:03 +0000 Subject: [PATCH 4/8] Let an operation select the algorithm and pin the request contract Read the algorithm through a resolver, so a per-operation option set as the name of an algorithm is accepted: a per-operation option is stored unparsed. Add the typed setters QuerySettings#compressionAlgorithm and InsertSettings#compressionAlgorithm. Warn when a request is compressed without http compression and another algorithm than LZ4 is selected: the ClickHouse framing of a request is LZ4. Pin the request contract with a mock-server test: compress=1 is not requested, a response is requested with the content coding of the algorithm, NONE requests no compression, and an operation overrides the client. --- CHANGELOG.md | 4 +- .../com/clickhouse/client/api/Client.java | 4 +- .../api/enums/CompressionAlgorithm.java | 4 +- .../client/api/insert/InsertSettings.java | 14 ++ .../api/internal/HttpAPIClientHelper.java | 35 +++-- .../client/api/query/QuerySettings.java | 14 ++ .../api/CompressionRequestUnitTest.java | 127 ++++++++++++++++++ docs/releases/0_11_0.md | 5 +- 8 files changed, 192 insertions(+), 15 deletions(-) create mode 100644 client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 18b11238d..acf9cd7be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,9 @@ now requested with the content coding of the new `client.compression_algorithm` property (`Client.Builder#compressionAlgorithm`), which defaults to `LZ4` and keeps the algorithm of a compressed body the same on every server version. Set the property to `ZSTD`, `GZIP` or `NONE` to select another algorithm; `ZSTD` - needs `com.github.luben:zstd-jni` on the classpath, which stays a `provided` dependency. A client that reads a + needs `com.github.luben:zstd-jni` on the classpath, which the client does not bring - the dependency of + `clickhouse-jdbc` stays `provided`, so packaging is unchanged and an application that selects `ZSTD` declares the + dependency itself. A client that reads a compressed response now also sends `enable_http_compression=1`, which a user profile that forbids setting changes (`readonly = 1`) rejects - such a profile has to use `readonly = 2` or `client.compression_algorithm = NONE`. (https://github.com/ClickHouse/clickhouse-java/issues/3105) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index f6762f3d6..e798b85e8 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -663,8 +663,8 @@ public Builder compressClientRequest(boolean enabled) { * never one the server picks on its own. {@link CompressionAlgorithm#NONE} disables compression. * Default is {@link CompressionAlgorithm#LZ4}. *

- * {@link CompressionAlgorithm#ZSTD} needs {@code com.github.luben:zstd-jni} on the classpath - the - * dependency is {@code provided}, so an application that selects the algorithm has to declare it. + * {@link CompressionAlgorithm#ZSTD} needs {@code com.github.luben:zstd-jni} on the classpath, which the + * client does not bring: an application that selects the algorithm declares the dependency itself. *

* A request body follows this algorithm only together with {@link #useHttpCompression(boolean)}; * the ClickHouse framing of a request compressed without it is always LZ4. diff --git a/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java b/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java index 9b79162a8..d9df4f26d 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java @@ -9,8 +9,8 @@ * *

    *
  • {@link #LZ4} - default. Needs {@code org.lz4:lz4-java}, which the client depends on.
    • - *
  • {@link #ZSTD} - needs {@code com.github.luben:zstd-jni} on the classpath. The dependency is - * {@code provided}, so an application that selects this algorithm has to declare it.
    • + *
  • {@link #ZSTD} - needs {@code com.github.luben:zstd-jni} on the classpath, which the client does not + * bring: an application that selects this algorithm declares the dependency itself.
    • *
  • {@link #GZIP} - supported by the JDK, so it needs no additional dependency.
    • *
  • {@link #NONE} - no compression, whatever {@code compress}/{@code decompress} are set to.
    • *
diff --git a/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java b/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java index 078019b17..0434bf25c 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java @@ -2,6 +2,7 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.enums.CompressionAlgorithm; import com.clickhouse.client.api.Session; import com.clickhouse.client.api.internal.CommonSettings; import org.apache.hc.core5.http.HttpHeaders; @@ -213,6 +214,19 @@ public InsertSettings compressClientRequest(boolean enabled) { return this; } + /** + * Algorithm of a compressed request or response body of this operation. The algorithm is requested with + * the HTTP content coding of the operation, so a compressed body always uses the algorithm set here. + * {@code CompressionAlgorithm.NONE} disables compression. Defaults to the algorithm of the client. + * + * @param algorithm - algorithm of a compressed body + * @return same instance of the settings + */ + public InsertSettings compressionAlgorithm(CompressionAlgorithm algorithm) { + settings.setOption(ClientConfigProperties.COMPRESSION_ALGORITHM.getKey(), algorithm); + return this; + } + public InsertSettings useHttpCompression(boolean enabled) { settings.setOption(ClientConfigProperties.USE_HTTP_COMPRESSION.getKey(), enabled); return this; diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index e944f9424..10ab559a0 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -163,7 +163,14 @@ public HttpAPIClientHelper(Map configuration, Object metricsRegi boolean usingServerCompression = ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getOrDefault(configuration); boolean useHttpCompression = ClientConfigProperties.USE_HTTP_COMPRESSION.getOrDefault(configuration); - LOG.debug("client compression: {}, server compression: {}, http compression: {}", usingClientCompression, usingServerCompression, useHttpCompression); + CompressionAlgorithm algorithm = compressionAlgorithm(configuration); + LOG.debug("client compression: {}, server compression: {}, http compression: {}, algorithm: {}", + usingClientCompression, usingServerCompression, useHttpCompression, algorithm); + if (usingClientCompression && !useHttpCompression + && !(algorithm == CompressionAlgorithm.LZ4 || algorithm == CompressionAlgorithm.NONE)) { + LOG.warn("Request compression uses LZ4 instead of {}: the ClickHouse framing of a request is LZ4 " + + "unless http compression is used", algorithm); + } defaultRetryCauses = new HashSet<>(ClientConfigProperties.CLIENT_RETRY_ON_FAILURE.getOrDefault(configuration)); if (defaultRetryCauses.contains(ClientFaultCause.None)) { @@ -774,9 +781,7 @@ private TransportResponse doExecuteRequest(TransportRequest transportRequest, Sp spanRecorder.recordHttpStatus(requestSpan, httpResponse.getCode()); } - httpResponse.setEntity(wrapResponseEntity(httpResponse.getEntity(), - httpResponse.getCode(), - requestConfig)); + httpResponse.setEntity(wrapResponseEntity(httpResponse.getEntity())); if (httpResponse.containsHeader(ClickHouseHttpProto.HEADER_EXCEPTION_CODE)) { throw readError(req, httpResponse); @@ -925,7 +930,7 @@ private void addHeaders(HttpPost req, Map requestConfig) { boolean serverCompression = ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getOrDefault(requestConfig); boolean useHttpCompression = ClientConfigProperties.USE_HTTP_COMPRESSION.getOrDefault(requestConfig); boolean appCompressedData = ClientConfigProperties.APP_COMPRESSED_DATA.getOrDefault(requestConfig); - CompressionAlgorithm algorithm = ClientConfigProperties.COMPRESSION_ALGORITHM.getOrDefault(requestConfig); + CompressionAlgorithm algorithm = compressionAlgorithm(requestConfig); if (algorithm != CompressionAlgorithm.NONE) { if (serverCompression) { @@ -974,7 +979,7 @@ private void addRequestParams(Map requestConfig, BiConsumer requestConfig, BiConsumer requestConfig) { + Object value = requestConfig.get(ClientConfigProperties.COMPRESSION_ALGORITHM.getKey()); + if (value == null) { + return ClientConfigProperties.COMPRESSION_ALGORITHM.getDefObjVal(); + } + return value instanceof CompressionAlgorithm + ? (CompressionAlgorithm) value + : CompressionAlgorithm.fromValue(String.valueOf(value)); + } + private HttpEntity wrapRequestEntity(HttpEntity httpEntity, Map requestConfig) { boolean clientCompression = ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getOrDefault(requestConfig); boolean useHttpCompression = ClientConfigProperties.USE_HTTP_COMPRESSION.getOrDefault(requestConfig); boolean appCompressedData = ClientConfigProperties.APP_COMPRESSED_DATA.getOrDefault(requestConfig); - CompressionAlgorithm algorithm = ClientConfigProperties.COMPRESSION_ALGORITHM.getOrDefault(requestConfig); + CompressionAlgorithm algorithm = compressionAlgorithm(requestConfig); if (httpEntity.getContentEncoding() != null && !appCompressedData) { // http header is set and data is not compressed @@ -1036,7 +1055,7 @@ private HttpEntity wrapRequestEntity(HttpEntity httpEntity, Map } } - private HttpEntity wrapResponseEntity(HttpEntity httpEntity, int httpStatus, Map requestConfig) { + private HttpEntity wrapResponseEntity(HttpEntity httpEntity) { if (httpEntity.getContentEncoding() != null) { // the algorithm of a compressed response is the one the request asked for return new CompressedEntity(httpEntity, true, CompressorStreamFactory.getSingleton()); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java b/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java index 9df39f407..04343bced 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java @@ -4,6 +4,7 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.Session; +import com.clickhouse.client.api.enums.CompressionAlgorithm; import com.clickhouse.client.api.internal.CommonSettings; import com.clickhouse.client.api.internal.ServerSettings; import com.clickhouse.client.api.internal.ValidationUtils; @@ -241,6 +242,19 @@ public TimeZone getServerTimeZone() { * @return same instance of the builder * @see Client.Builder#httpHeaders(Map) */ + /** + * Algorithm of a compressed response body of this operation. The algorithm is requested with the HTTP + * content coding of the operation, so a compressed body always uses the algorithm set here. + * {@link CompressionAlgorithm#NONE} disables compression. Defaults to the algorithm of the client. + * + * @param algorithm - algorithm of a compressed body + * @return same instance of the settings + */ + public QuerySettings compressionAlgorithm(CompressionAlgorithm algorithm) { + settings.setOption(ClientConfigProperties.COMPRESSION_ALGORITHM.getKey(), algorithm); + return this; + } + public QuerySettings httpHeader(String key, String value) { settings.httpHeader(key, value); return this; diff --git a/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java new file mode 100644 index 000000000..c9867ef9a --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java @@ -0,0 +1,127 @@ +package com.clickhouse.client.api; + +import com.clickhouse.client.api.enums.CompressionAlgorithm; +import com.clickhouse.client.api.enums.Protocol; +import com.clickhouse.client.api.query.QuerySettings; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.common.ConsoleNotifier; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.http.Request; +import com.github.tomakehurst.wiremock.verification.LoggedRequest; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.List; + +public class CompressionRequestUnitTest { + + private WireMockServer server; + + @BeforeClass(groups = {"unit"}) + public void startServer() { + server = new WireMockServer(WireMockConfiguration.options() + .dynamicPort() + .notifier(new ConsoleNotifier(false))); + server.start(); + server.addStubMapping(WireMock.post(WireMock.anyUrl()) + .willReturn(WireMock.aResponse().withBody("")).build()); + } + + @AfterClass(groups = {"unit"}) + public void stopServer() { + if (server != null) { + server.stop(); + } + } + + @DataProvider(name = "responseCompressionRequests") + public static Object[][] responseCompressionRequests() { + return new Object[][]{ + // algorithm -> content coding the response is requested with (null: no compression requested) + {CompressionAlgorithm.LZ4, "lz4"}, + {CompressionAlgorithm.ZSTD, "zstd"}, + {CompressionAlgorithm.GZIP, "gzip"}, + {CompressionAlgorithm.NONE, null}, + }; + } + + @Test(groups = {"unit"}, dataProvider = "responseCompressionRequests") + public void testResponseCompressionRequestedWithContentCoding(CompressionAlgorithm algorithm, String coding) { + LoggedRequest request = runQuery(builder -> builder.compressionAlgorithm(algorithm), null); + + Assert.assertEquals(header(request, "Accept-Encoding"), coding); + // the codec of the compress=1 framing is the one of the server, so it is never requested + Assert.assertFalse(request.queryParameter("compress").isPresent(), + "compress=1 must not be requested"); + Assert.assertEquals(request.queryParameter("enable_http_compression").isPresent(), coding != null); + } + + @Test(groups = {"unit"}) + public void testAlgorithmDefaultsToLz4() { + LoggedRequest request = runQuery(builder -> builder, null); + + Assert.assertEquals(header(request, "Accept-Encoding"), "lz4"); + Assert.assertFalse(request.queryParameter("compress").isPresent()); + } + + @Test(groups = {"unit"}) + public void testOperationOverridesClientAlgorithm() { + LoggedRequest request = runQuery(builder -> builder.compressionAlgorithm(CompressionAlgorithm.LZ4), + new QuerySettings().compressionAlgorithm(CompressionAlgorithm.GZIP)); + + Assert.assertEquals(header(request, "Accept-Encoding"), "gzip"); + } + + @Test(groups = {"unit"}) + public void testOperationAcceptsAlgorithmName() { + LoggedRequest request = runQuery(builder -> builder, + (QuerySettings) new QuerySettings() + .setOption(ClientConfigProperties.COMPRESSION_ALGORITHM.getKey(), "zstd")); + + Assert.assertEquals(header(request, "Accept-Encoding"), "zstd"); + } + + @Test(groups = {"unit"}) + public void testRequestCompressedWithContentCodingOfAlgorithm() { + LoggedRequest request = runQuery(builder -> builder + .compressionAlgorithm(CompressionAlgorithm.GZIP) + .compressClientRequest(true) + .useHttpCompression(true), null); + + Assert.assertEquals(header(request, "Content-Encoding"), "gzip"); + } + + private LoggedRequest runQuery(java.util.function.UnaryOperator configure, + QuerySettings settings) { + server.resetRequests(); + Client.Builder builder = new Client.Builder() + .addEndpoint(Protocol.HTTP, "localhost", server.port(), false) + .setUsername("default") + .setPassword("") + .retryOnFailures(); + + try (Client client = configure.apply(builder).build()) { + try { + if (settings == null) { + client.query("SELECT 1").get(); + } else { + client.query("SELECT 1", settings).get(); + } + } catch (Exception e) { + // the stub answers an empty body, so only the request itself is of interest here + } + } + + List requests = server.findAll(WireMock.postRequestedFor(WireMock.anyUrl())); + Assert.assertEquals(requests.size(), 1, "expected exactly one request"); + return requests.get(0); + } + + private static String header(Request request, String name) { + return request.containsHeader(name) ? request.getHeader(name) : null; + } +} diff --git a/docs/releases/0_11_0.md b/docs/releases/0_11_0.md index 682e90089..443dab628 100644 --- a/docs/releases/0_11_0.md +++ b/docs/releases/0_11_0.md @@ -16,8 +16,9 @@ a compressed body is always the one the client asked for: algorithm out of `LZ4` (default), `ZSTD`, `GZIP` and `NONE`. Both the name and the content-coding token of an algorithm are accepted, in any case. - The default is `LZ4`, so an application that does not set the property keeps reading `LZ4` on every server version. -- `ZSTD` needs `com.github.luben:zstd-jni` on the classpath. The dependency stays `provided`, so an application that - selects `ZSTD` has to declare it. +- `ZSTD` needs `com.github.luben:zstd-jni` on the classpath, which the client does not bring: the dependency of + `clickhouse-jdbc` stays `provided`, so packaging is unchanged and an application that selects `ZSTD` declares the + dependency itself. - `NONE` disables compression of the request and the response, whatever `compress` and `decompress` are set to. What to check in an application: From d34138194ea38df14e289d46463c95a0029e977a Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Tue, 8 Sep 2026 19:40:25 +0000 Subject: [PATCH 5/8] Count the new default setting and cover the request contract of an operation The client now seeds one more default setting, so the canary counts of ClientTests, which the test itself asks to increment when a setting is added, move to the new size. They were the whole failure of the client-v2 legs: the failing module stopped before the JaCoCo merge of the coverage profile, which reports the coverage of new code as zero. Two paths of the new setting had no test: an insert that selects its own algorithm, and a request compressed without http compression, whose framing stays the ClickHouse one while the response still follows the algorithm. The javadoc of httpHeader(String, String) is restored - the new method of QuerySettings was inserted between the javadoc and its method. --- .../client/api/query/QuerySettings.java | 18 ++--- .../com/clickhouse/client/ClientTests.java | 6 +- .../api/CompressionRequestUnitTest.java | 68 ++++++++++++++++--- 3 files changed, 72 insertions(+), 20 deletions(-) diff --git a/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java b/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java index 04343bced..100694dac 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java @@ -233,15 +233,6 @@ public TimeZone getServerTimeZone() { return (TimeZone) settings.getOption(ClientConfigProperties.SERVER_TIMEZONE.getKey()); } - /** - * Defines list of headers that should be sent with current request. The Client will use a header value - * defined in {@code headers} instead of any other. - * - * @param key - header name. - * @param value - header value. - * @return same instance of the builder - * @see Client.Builder#httpHeaders(Map) - */ /** * Algorithm of a compressed response body of this operation. The algorithm is requested with the HTTP * content coding of the operation, so a compressed body always uses the algorithm set here. @@ -255,6 +246,15 @@ public QuerySettings compressionAlgorithm(CompressionAlgorithm algorithm) { return this; } + /** + * Defines list of headers that should be sent with current request. The Client will use a header value + * defined in {@code headers} instead of any other. + * + * @param key - header name. + * @param value - header value. + * @return same instance of the builder + * @see Client.Builder#httpHeaders(Map) + */ public QuerySettings httpHeader(String key, String value) { settings.httpHeader(key, value); return this; diff --git a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java index b50d3700b..a9c2342f8 100644 --- a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java @@ -333,7 +333,7 @@ public void testDefaultSettings() { Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); } } - Assert.assertEquals(config.size(), 37); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added. } try (Client client = new Client.Builder() @@ -367,7 +367,7 @@ public void testDefaultSettings() { .binaryStringSupport(true) .build()) { Map config = client.getConfiguration(); - Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 39); // to check everything is set. Increment when new added. Assert.assertEquals(config.get(ClientConfigProperties.DATABASE.getKey()), "mydb"); Assert.assertEquals(config.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey()), "10"); Assert.assertEquals(config.get(ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getKey()), "300000"); @@ -437,7 +437,7 @@ public void testWithOldDefaults() { Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); } } - Assert.assertEquals(config.size(), 37); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added. } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java index c9867ef9a..a722e1693 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java @@ -2,7 +2,9 @@ import com.clickhouse.client.api.enums.CompressionAlgorithm; import com.clickhouse.client.api.enums.Protocol; +import com.clickhouse.client.api.insert.InsertSettings; import com.clickhouse.client.api.query.QuerySettings; +import com.clickhouse.data.ClickHouseFormat; import com.github.tomakehurst.wiremock.WireMockServer; import com.github.tomakehurst.wiremock.client.WireMock; import com.github.tomakehurst.wiremock.common.ConsoleNotifier; @@ -15,7 +17,10 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.function.UnaryOperator; public class CompressionRequestUnitTest { @@ -95,16 +100,51 @@ public void testRequestCompressedWithContentCodingOfAlgorithm() { Assert.assertEquals(header(request, "Content-Encoding"), "gzip"); } - private LoggedRequest runQuery(java.util.function.UnaryOperator configure, - QuerySettings settings) { + @Test(groups = {"unit"}) + public void testRequestFramingStaysLz4WithoutHttpCompression() { + LoggedRequest request = runQuery(builder -> builder + .compressionAlgorithm(CompressionAlgorithm.GZIP) + .compressClientRequest(true) + .useHttpCompression(false), null); + + // the ClickHouse framing of a request is LZ4, so the algorithm applies to the response only + Assert.assertNull(header(request, "Content-Encoding")); + Assert.assertEquals(header(request, "Accept-Encoding"), "gzip"); + Assert.assertTrue(request.queryParameter("enable_http_compression").isPresent(), + "the response is compressed with the requested content coding"); + Assert.assertFalse(request.queryParameter("compress").isPresent(), + "compress=1 must not be requested"); + Assert.assertTrue(request.queryParameter("decompress").isPresent(), + "a request compressed without http compression keeps the ClickHouse framing"); + } + + @Test(groups = {"unit"}) + public void testInsertOperationOverridesClientAlgorithm() { server.resetRequests(); - Client.Builder builder = new Client.Builder() - .addEndpoint(Protocol.HTTP, "localhost", server.port(), false) - .setUsername("default") - .setPassword("") - .retryOnFailures(); + try (Client client = newBuilder() + .compressionAlgorithm(CompressionAlgorithm.LZ4) + .compressClientRequest(true) + .useHttpCompression(true) + .build()) { + try { + client.insert("some_table", + new ByteArrayInputStream("1\n".getBytes(StandardCharsets.UTF_8)), + ClickHouseFormat.TSV, + new InsertSettings().compressionAlgorithm(CompressionAlgorithm.GZIP)).get(); + } catch (Exception e) { + // the stub answers an empty body, so only the request itself is of interest here + } + } - try (Client client = configure.apply(builder).build()) { + LoggedRequest request = lastRequest(); + Assert.assertEquals(header(request, "Content-Encoding"), "gzip"); + Assert.assertEquals(header(request, "Accept-Encoding"), "gzip"); + } + + private LoggedRequest runQuery(UnaryOperator configure, + QuerySettings settings) { + server.resetRequests(); + try (Client client = configure.apply(newBuilder()).build()) { try { if (settings == null) { client.query("SELECT 1").get(); @@ -116,6 +156,18 @@ private LoggedRequest runQuery(java.util.function.UnaryOperator } } + return lastRequest(); + } + + private Client.Builder newBuilder() { + return new Client.Builder() + .addEndpoint(Protocol.HTTP, "localhost", server.port(), false) + .setUsername("default") + .setPassword("") + .retryOnFailures(); + } + + private LoggedRequest lastRequest() { List requests = server.findAll(WireMock.postRequestedFor(WireMock.anyUrl())); Assert.assertEquals(requests.size(), 1, "expected exactly one request"); return requests.get(0); From b776b5d4378855c34faaf99f3836b1c8fbe7fb1e Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:10:17 +0000 Subject: [PATCH 6/8] test(client-v2): bump ClientTests default-settings canary counts to 39/40/39 main bumped the same canary constants to 38/39/38 for a default property added on its side. The two changes are textually identical, so the merge kept one copy while both properties exist, leaving the expectation one short of the real configuration size. --- .../src/test/java/com/clickhouse/client/ClientTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java index aef988818..45feadd70 100644 --- a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java @@ -333,7 +333,7 @@ public void testDefaultSettings() { Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); } } - Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 39); // to check everything is set. Increment when new added. } try (Client client = new Client.Builder() @@ -368,7 +368,7 @@ public void testDefaultSettings() { .queryFormat(ClickHouseFormat.CSV.name()) .build()) { Map config = client.getConfiguration(); - Assert.assertEquals(config.size(), 39); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 40); // to check everything is set. Increment when new added. Assert.assertEquals(config.get(ClientConfigProperties.DATABASE.getKey()), "mydb"); Assert.assertEquals(config.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey()), "10"); Assert.assertEquals(config.get(ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getKey()), "300000"); @@ -438,7 +438,7 @@ public void testWithOldDefaults() { Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); } } - Assert.assertEquals(config.size(), 38); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 39); // to check everything is set. Increment when new added. } } From bae32e4b477040845777b91aa4b677aa222cee21 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:30:24 +0000 Subject: [PATCH 7/8] test(client-v2): align the merged request-compression signalling expectations #3116 landed a data-driven test of the request compression signalling while this branch changes how a compressed response is requested: a response is now asked for with a content coding (Accept-Encoding + enable_http_compression=1) instead of the compress=1 framing, so the codec is the one the client selects. Two rows of the merged data provider still pinned the previous signalling. The rows now state the response-compression expectations of the new behaviour, the enable_http_compression expectation becomes an explicit column instead of a copy of useHttpCompression, and a row with response compression disabled keeps the assertion discriminating. The multipart intent of the test is unchanged. --- .../api/internal/HttpAPIClientHelperTest.java | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java index 0c7fa375a..519e2c297 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpAPIClientHelperTest.java @@ -332,28 +332,34 @@ public void testShouldRetryUsesServerExceptionFromCause(Throwable ex, boolean ex /** * A multipart body (statement parameters sent as form data) is never compressed, so the request must not * declare a content encoding - the server would try to decompress the plain body and fail with - * LZ4_DECODER_FAILED. A request that is not multipart, and response compression, keep their signalling. + * LZ4_DECODER_FAILED. A request that is not multipart, and response compression, keep their signalling: + * a compressed response is always asked for with a content coding, so its codec is the one the client + * selects, whatever the form of the request body is. */ @DataProvider(name = "requestCompressionSignalling") public static Object[][] requestCompressionSignalling() { return new Object[][] { - // clientCompression, useHttpCompression, sendParamsInBody, withParams, - // contentEncoding, acceptEncoding, decompressParam - {true, true, true, true, null, "lz4", false}, - {true, true, true, false, "lz4", "lz4", false}, // no parameters -> not a multipart request - {true, true, false, true, "lz4", "lz4", false}, - {false, true, true, true, null, "lz4", false}, - {true, false, true, true, null, null, false}, - {true, false, false, true, null, null, true}, + // clientCompression, useHttpCompression, serverCompression, sendParamsInBody, withParams, + // contentEncoding, acceptEncoding, decompressParam, httpCompressionParam + {true, true, true, true, true, null, "lz4", false, true}, + {true, true, true, true, false, "lz4", "lz4", false, true}, // no parameters -> not a multipart request + {true, true, true, false, true, "lz4", "lz4", false, true}, + {false, true, true, true, true, null, "lz4", false, true}, + {true, false, true, true, true, null, "lz4", false, true}, + {true, false, true, false, true, null, "lz4", true, true}, + // no response compression -> nothing is signalled for it; the request body keeps its own + {true, false, false, false, true, null, null, true, false}, }; } @Test(dataProvider = "requestCompressionSignalling") public void testRequestCompressionSignalling(boolean clientCompression, boolean useHttpCompression, - boolean sendParamsInBody, boolean withParams, - String expectedContentEncoding, String expectedAcceptEncoding, - boolean expectDecompressParam) { + boolean serverCompression, boolean sendParamsInBody, + boolean withParams, String expectedContentEncoding, + String expectedAcceptEncoding, boolean expectDecompressParam, + boolean expectHttpCompressionParam) { Map reqConfig = compressionConfig(clientCompression, useHttpCompression, sendParamsInBody); + reqConfig.put(ClientConfigProperties.COMPRESS_SERVER_RESPONSE.getKey(), serverCompression); if (withParams) { reqConfig.put(HttpAPIClientHelper.KEY_STATEMENT_PARAMS, Collections.singletonMap("p1", "1")); } @@ -362,18 +368,20 @@ public void testRequestCompressionSignalling(boolean clientCompression, boolean "SELECT {p1:Int32}").getDelegate(); String setup = "clientCompression=" + clientCompression + ", useHttpCompression=" + useHttpCompression - + ", sendParamsInBody=" + sendParamsInBody + ", withParams=" + withParams; + + ", serverCompression=" + serverCompression + ", sendParamsInBody=" + sendParamsInBody + + ", withParams=" + withParams; assertEquals(headerValue(req, HttpHeaders.CONTENT_ENCODING), expectedContentEncoding, "unexpected " + HttpHeaders.CONTENT_ENCODING + " for " + setup); assertEquals(req.getEntity().getContentEncoding(), expectedContentEncoding, "the request body entity must declare the same encoding as the request for " + setup); assertEquals(headerValue(req, HttpHeaders.ACCEPT_ENCODING), expectedAcceptEncoding, - "response compression signalling must not depend on the request body form"); + "response compression signalling must not depend on the request body form, for " + setup); String query = req.getRequestUri(); assertEquals(query.contains(ClickHouseHttpProto.QPARAM_DECOMPRESS + "=1"), expectDecompressParam, "unexpected " + ClickHouseHttpProto.QPARAM_DECOMPRESS + " parameter in " + query); - assertEquals(query.contains(ClickHouseHttpProto.QPARAM_ENABLE_HTTP_COMPRESSION + "=1"), useHttpCompression, + assertEquals(query.contains(ClickHouseHttpProto.QPARAM_ENABLE_HTTP_COMPRESSION + "=1"), + expectHttpCompressionParam, "unexpected " + ClickHouseHttpProto.QPARAM_ENABLE_HTTP_COMPRESSION + " parameter in " + query); } From 5038993edf6295af4a89cf6b8810a1079baaa308 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Fri, 11 Sep 2026 00:38:55 +0000 Subject: [PATCH 8/8] fix(client-v2): keep only LZ4 and ZSTD in CompressionAlgorithm and bring zstd-jni Review feedback on #3106: the algorithm of a compressed body and the decision to compress it are two separate concerns. The enum carried GZIP and NONE, so NONE duplicated the compression flags and every header/parameter of the compression logic had to be gated on it. The enum now holds only the two algorithms ClickHouse frames a body with, and whether a body is compressed stays with compressServerResponse / compressClientRequest / useHttpCompression alone. zstd-jni was a test-scoped dependency of client-v2 and a provided dependency of clickhouse-jdbc, so an application that selected ZSTD had to declare it itself. It is a normal dependency of both now, so the shaded jars carry it. Tests: the LZ4/ZSTD matrix keeps its coverage; the removed NONE case becomes an explicit flags-disable-compression case (unit + integration), and an integration case pins the algorithm together with http compression. --- CHANGELOG.md | 13 ++--- clickhouse-jdbc/pom.xml | 1 - client-v2/pom.xml | 11 ++-- .../com/clickhouse/client/api/Client.java | 7 ++- .../client/api/ClientConfigProperties.java | 4 +- .../api/enums/CompressionAlgorithm.java | 23 +++----- .../client/api/insert/InsertSettings.java | 3 +- .../api/internal/HttpAPIClientHelper.java | 52 ++++++++----------- .../client/api/query/QuerySettings.java | 3 +- .../api/ClientConfigPropertiesTest.java | 4 -- .../api/CompressionRequestUnitTest.java | 37 ++++++++----- .../QueryServerContentCompressionTests.java | 19 ++++++- .../QueryServerHttpCompressionTests.java | 25 +++++++++ docs/features.md | 2 +- docs/releases/0_11_0.md | 14 ++--- 15 files changed, 125 insertions(+), 93 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6757cafad..c013efb47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,13 @@ default and no setting overrides it, so the client could not keep reading a response it asked for. A response is now requested with the content coding of the new `client.compression_algorithm` property (`Client.Builder#compressionAlgorithm`), which defaults to `LZ4` and keeps the algorithm of a compressed body the - same on every server version. Set the property to `ZSTD`, `GZIP` or `NONE` to select another algorithm; `ZSTD` - needs `com.github.luben:zstd-jni` on the classpath, which the client does not bring - the dependency of - `clickhouse-jdbc` stays `provided`, so packaging is unchanged and an application that selects `ZSTD` declares the - dependency itself. A client that reads a - compressed response now also sends `enable_http_compression=1`, which a user profile that forbids setting changes - (`readonly = 1`) rejects - such a profile has to use `readonly = 2` or `client.compression_algorithm = NONE`. + same on every server version. Set the property to `ZSTD` to select the other algorithm; the client now brings + `com.github.luben:zstd-jni` itself, and the dependency of `clickhouse-jdbc` is no longer `provided`, so the shaded + jars carry it and an application needs no additional dependency. The property selects only *how* a body is + compressed - *whether* it is compressed stays with `compressServerResponse` and `compressClientRequest`. A client + that reads a compressed response now also sends `enable_http_compression=1`, which a user profile that forbids + setting changes (`readonly = 1`) rejects - such a profile has to use `readonly = 2` or read an uncompressed + response (`compressServerResponse(false)`). (https://github.com/ClickHouse/clickhouse-java/issues/3105) - **[client-v2]** `com.clickhouse.client.api.metrics.OperationMetrics` now has a single constructor, diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml index c4c56727f..9707bfccb 100644 --- a/clickhouse-jdbc/pom.xml +++ b/clickhouse-jdbc/pom.xml @@ -61,7 +61,6 @@ com.github.luben zstd-jni - provided org.slf4j diff --git a/client-v2/pom.xml b/client-v2/pom.xml index c1b4e92d2..c0f37c3b2 100644 --- a/client-v2/pom.xml +++ b/client-v2/pom.xml @@ -58,6 +58,11 @@ ${lz4.version} + + com.github.luben + zstd-jni + + org.apache.commons commons-compress @@ -188,12 +193,6 @@ 5.19.0 test - - com.github.luben - zstd-jni - 1.5.7-6 - test - org.bouncycastle bcprov-jdk18on diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 92b641209..d609e050f 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -663,11 +663,10 @@ public Builder compressClientRequest(boolean enabled) { /** * Algorithm of a compressed request or response body. The algorithm is requested with the HTTP * content-coding of the operation, so a compressed body always uses the algorithm set here and - * never one the server picks on its own. {@link CompressionAlgorithm#NONE} disables compression. - * Default is {@link CompressionAlgorithm#LZ4}. + * never one the server picks on its own. Default is {@link CompressionAlgorithm#LZ4}. *

- * {@link CompressionAlgorithm#ZSTD} needs {@code com.github.luben:zstd-jni} on the classpath, which the - * client does not bring: an application that selects the algorithm declares the dependency itself. + * The algorithm selects only how a body is compressed. Whether a body is compressed is controlled by + * {@link #compressServerResponse(boolean)} and {@link #compressClientRequest(boolean)}. *

* A request body follows this algorithm only together with {@link #useHttpCompression(boolean)}; * the ClickHouse framing of a request compressed without it is always LZ4. diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index daede437d..77a765a7e 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -251,8 +251,8 @@ public Object parseValue(String value) { * Algorithm of a compressed request or response body. The algorithm is requested with the HTTP * content-coding of the operation ({@code Accept-Encoding} for a response, {@code Content-Encoding} * for a request), so a compressed body always uses the algorithm the client asked for and never one - * the server picks on its own. {@link CompressionAlgorithm#NONE} disables compression of both - * directions. + * the server picks on its own. The algorithm selects only how a body is compressed; whether a body is + * compressed is controlled by {@link #COMPRESS_SERVER_RESPONSE} and {@link #COMPRESS_CLIENT_REQUEST}. *

* The name of an algorithm and its content-coding token are both accepted, in any case. *

diff --git a/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java b/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java index d9df4f26d..467491f3d 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/enums/CompressionAlgorithm.java @@ -7,12 +7,13 @@ * for a response and {@code Content-Encoding} for a request - so a compressed body always uses the * algorithm of the request and never one the server picks on its own.

* + *

The algorithm selects only how a body is compressed. Whether a body is compressed is + * controlled by the compression flags - {@code compressServerResponse} for a response and + * {@code compressClientRequest} for a request.

+ * *
    *
  • {@link #LZ4} - default. Needs {@code org.lz4:lz4-java}, which the client depends on.
    • - *
  • {@link #ZSTD} - needs {@code com.github.luben:zstd-jni} on the classpath, which the client does not - * bring: an application that selects this algorithm declares the dependency itself.
    • - *
  • {@link #GZIP} - supported by the JDK, so it needs no additional dependency.
    • - *
  • {@link #NONE} - no compression, whatever {@code compress}/{@code decompress} are set to.
    • + *
  • {@link #ZSTD} - needs {@code com.github.luben:zstd-jni}, which the client depends on.
    • *
*/ public enum CompressionAlgorithm { @@ -23,19 +24,9 @@ public enum CompressionAlgorithm { LZ4("lz4"), /** - * Zstandard. Requires {@code com.github.luben:zstd-jni} on the classpath. - */ - ZSTD("zstd"), - - /** - * gzip. Supported by the JDK. - */ - GZIP("gzip"), - - /** - * No compression. + * Zstandard. */ - NONE("none"); + ZSTD("zstd"); private final String httpContentCoding; diff --git a/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java b/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java index 0434bf25c..d8597b0f8 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/insert/InsertSettings.java @@ -217,7 +217,8 @@ public InsertSettings compressClientRequest(boolean enabled) { /** * Algorithm of a compressed request or response body of this operation. The algorithm is requested with * the HTTP content coding of the operation, so a compressed body always uses the algorithm set here. - * {@code CompressionAlgorithm.NONE} disables compression. Defaults to the algorithm of the client. + * Whether the body is compressed is controlled by the compression flags of the client. Defaults to + * the algorithm of the client. * * @param algorithm - algorithm of a compressed body * @return same instance of the settings diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 5050fc746..bf6fd882d 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -166,8 +166,7 @@ public HttpAPIClientHelper(Map configuration, Object metricsRegi CompressionAlgorithm algorithm = compressionAlgorithm(configuration); LOG.debug("client compression: {}, server compression: {}, http compression: {}, algorithm: {}", usingClientCompression, usingServerCompression, useHttpCompression, algorithm); - if (usingClientCompression && !useHttpCompression - && !(algorithm == CompressionAlgorithm.LZ4 || algorithm == CompressionAlgorithm.NONE)) { + if (usingClientCompression && !useHttpCompression && algorithm != CompressionAlgorithm.LZ4) { LOG.warn("Request compression uses LZ4 instead of {}: the ClickHouse framing of a request is LZ4 " + "unless http compression is used", algorithm); } @@ -943,16 +942,14 @@ private void addHeaders(HttpPost req, Map requestConfig) { boolean appCompressedData = ClientConfigProperties.APP_COMPRESSED_DATA.getOrDefault(requestConfig); CompressionAlgorithm algorithm = compressionAlgorithm(requestConfig); - if (algorithm != CompressionAlgorithm.NONE) { - if (serverCompression) { - // the codec of a compressed response is the one requested here: the server picks its own - // default codec for the compress=1 framing and does not let a client select it - setHeader(req, HttpHeaders.ACCEPT_ENCODING, algorithm.getHttpContentCoding()); - } + if (serverCompression) { + // the codec of a compressed response is the one requested here: the server picks its own + // default codec for the compress=1 framing and does not let a client select it + setHeader(req, HttpHeaders.ACCEPT_ENCODING, algorithm.getHttpContentCoding()); + } - if (useHttpCompression && clientCompression && !appCompressedData) { - setHeader(req, HttpHeaders.CONTENT_ENCODING, algorithm.getHttpContentCoding()); - } + if (useHttpCompression && clientCompression && !appCompressedData) { + setHeader(req, HttpHeaders.CONTENT_ENCODING, algorithm.getHttpContentCoding()); } for (String key : requestConfig.keySet()) { @@ -990,24 +987,20 @@ private void addRequestParams(Map requestConfig, BiConsumer boolean clientCompression = ClientConfigProperties.COMPRESS_CLIENT_REQUEST.getOrDefault(requestConfig); boolean useHttpCompression = ClientConfigProperties.USE_HTTP_COMPRESSION.getOrDefault(requestConfig); boolean appCompressedData = ClientConfigProperties.APP_COMPRESSED_DATA.getOrDefault(requestConfig); - CompressionAlgorithm algorithm = compressionAlgorithm(requestConfig); if (httpEntity.getContentEncoding() != null && !appCompressedData) { // http header is set and data is not compressed return new CompressedEntity(httpEntity, false, CompressorStreamFactory.getSingleton()); - } else if (clientCompression && !appCompressedData && algorithm != CompressionAlgorithm.NONE) { + } else if (clientCompression && !appCompressedData) { int buffSize = ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getOrDefault(requestConfig); return new LZ4Entity(httpEntity, useHttpCompression, false, true, buffSize, false, lz4Factory); diff --git a/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java b/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java index 100694dac..c2fb025d7 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/query/QuerySettings.java @@ -236,7 +236,8 @@ public TimeZone getServerTimeZone() { /** * Algorithm of a compressed response body of this operation. The algorithm is requested with the HTTP * content coding of the operation, so a compressed body always uses the algorithm set here. - * {@link CompressionAlgorithm#NONE} disables compression. Defaults to the algorithm of the client. + * Whether the response is compressed is controlled by the compression flags of the client. Defaults + * to the algorithm of the client. * * @param algorithm - algorithm of a compressed body * @return same instance of the settings diff --git a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java index 43991c051..28ed1e624 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/ClientConfigPropertiesTest.java @@ -86,10 +86,6 @@ public static Object[][] compressionAlgorithms() { {"lz4", CompressionAlgorithm.LZ4}, {"ZSTD", CompressionAlgorithm.ZSTD}, {"zstd", CompressionAlgorithm.ZSTD}, - {"GZIP", CompressionAlgorithm.GZIP}, - {"gzip", CompressionAlgorithm.GZIP}, - {"NONE", CompressionAlgorithm.NONE}, - {"none", CompressionAlgorithm.NONE}, }; } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java b/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java index a722e1693..0e46861d9 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/CompressionRequestUnitTest.java @@ -49,8 +49,6 @@ public static Object[][] responseCompressionRequests() { // algorithm -> content coding the response is requested with (null: no compression requested) {CompressionAlgorithm.LZ4, "lz4"}, {CompressionAlgorithm.ZSTD, "zstd"}, - {CompressionAlgorithm.GZIP, "gzip"}, - {CompressionAlgorithm.NONE, null}, }; } @@ -62,7 +60,22 @@ public void testResponseCompressionRequestedWithContentCoding(CompressionAlgorit // the codec of the compress=1 framing is the one of the server, so it is never requested Assert.assertFalse(request.queryParameter("compress").isPresent(), "compress=1 must not be requested"); - Assert.assertEquals(request.queryParameter("enable_http_compression").isPresent(), coding != null); + Assert.assertTrue(request.queryParameter("enable_http_compression").isPresent()); + } + + @Test(groups = {"unit"}) + public void testCompressionFlagsDisableCompression() { + LoggedRequest request = runQuery(builder -> builder + .compressionAlgorithm(CompressionAlgorithm.ZSTD) + .compressServerResponse(false) + .compressClientRequest(false), null); + + // the algorithm selects only how a body is compressed - the flags select whether it is + Assert.assertNull(header(request, "Accept-Encoding")); + Assert.assertNull(header(request, "Content-Encoding")); + Assert.assertFalse(request.queryParameter("enable_http_compression").isPresent()); + Assert.assertFalse(request.queryParameter("compress").isPresent()); + Assert.assertFalse(request.queryParameter("decompress").isPresent()); } @Test(groups = {"unit"}) @@ -76,9 +89,9 @@ public void testAlgorithmDefaultsToLz4() { @Test(groups = {"unit"}) public void testOperationOverridesClientAlgorithm() { LoggedRequest request = runQuery(builder -> builder.compressionAlgorithm(CompressionAlgorithm.LZ4), - new QuerySettings().compressionAlgorithm(CompressionAlgorithm.GZIP)); + new QuerySettings().compressionAlgorithm(CompressionAlgorithm.ZSTD)); - Assert.assertEquals(header(request, "Accept-Encoding"), "gzip"); + Assert.assertEquals(header(request, "Accept-Encoding"), "zstd"); } @Test(groups = {"unit"}) @@ -93,23 +106,23 @@ public void testOperationAcceptsAlgorithmName() { @Test(groups = {"unit"}) public void testRequestCompressedWithContentCodingOfAlgorithm() { LoggedRequest request = runQuery(builder -> builder - .compressionAlgorithm(CompressionAlgorithm.GZIP) + .compressionAlgorithm(CompressionAlgorithm.ZSTD) .compressClientRequest(true) .useHttpCompression(true), null); - Assert.assertEquals(header(request, "Content-Encoding"), "gzip"); + Assert.assertEquals(header(request, "Content-Encoding"), "zstd"); } @Test(groups = {"unit"}) public void testRequestFramingStaysLz4WithoutHttpCompression() { LoggedRequest request = runQuery(builder -> builder - .compressionAlgorithm(CompressionAlgorithm.GZIP) + .compressionAlgorithm(CompressionAlgorithm.ZSTD) .compressClientRequest(true) .useHttpCompression(false), null); // the ClickHouse framing of a request is LZ4, so the algorithm applies to the response only Assert.assertNull(header(request, "Content-Encoding")); - Assert.assertEquals(header(request, "Accept-Encoding"), "gzip"); + Assert.assertEquals(header(request, "Accept-Encoding"), "zstd"); Assert.assertTrue(request.queryParameter("enable_http_compression").isPresent(), "the response is compressed with the requested content coding"); Assert.assertFalse(request.queryParameter("compress").isPresent(), @@ -130,15 +143,15 @@ public void testInsertOperationOverridesClientAlgorithm() { client.insert("some_table", new ByteArrayInputStream("1\n".getBytes(StandardCharsets.UTF_8)), ClickHouseFormat.TSV, - new InsertSettings().compressionAlgorithm(CompressionAlgorithm.GZIP)).get(); + new InsertSettings().compressionAlgorithm(CompressionAlgorithm.ZSTD)).get(); } catch (Exception e) { // the stub answers an empty body, so only the request itself is of interest here } } LoggedRequest request = lastRequest(); - Assert.assertEquals(header(request, "Content-Encoding"), "gzip"); - Assert.assertEquals(header(request, "Accept-Encoding"), "gzip"); + Assert.assertEquals(header(request, "Content-Encoding"), "zstd"); + Assert.assertEquals(header(request, "Accept-Encoding"), "zstd"); } private LoggedRequest runQuery(UnaryOperator configure, diff --git a/client-v2/src/test/java/com/clickhouse/client/query/QueryServerContentCompressionTests.java b/client-v2/src/test/java/com/clickhouse/client/query/QueryServerContentCompressionTests.java index 9dc385318..521331f94 100644 --- a/client-v2/src/test/java/com/clickhouse/client/query/QueryServerContentCompressionTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/query/QueryServerContentCompressionTests.java @@ -28,13 +28,28 @@ public void testQueryWithCompressionAlgorithm(CompressionAlgorithm algorithm) th } } + @Test(groups = {"integration"}) + public void testQueryWithoutCompression() throws Exception { + // the algorithm selects only how a body is compressed - the flags select whether it is + try (Client client = newClient() + .compressionAlgorithm(CompressionAlgorithm.ZSTD) + .compressServerResponse(false) + .compressClientRequest(false) + .build()) { + List records = client.queryAll("SELECT number, toString(number) AS str " + + "FROM system.numbers LIMIT 1000"); + + Assert.assertEquals(records.size(), 1000); + Assert.assertEquals(records.get(999).getLong("number"), 999); + Assert.assertEquals(records.get(999).getString("str"), "999"); + } + } + @DataProvider(name = "compressionAlgorithms") public Object[][] compressionAlgorithms() { return new Object[][]{ {CompressionAlgorithm.LZ4}, {CompressionAlgorithm.ZSTD}, - {CompressionAlgorithm.GZIP}, - {CompressionAlgorithm.NONE}, }; } } diff --git a/client-v2/src/test/java/com/clickhouse/client/query/QueryServerHttpCompressionTests.java b/client-v2/src/test/java/com/clickhouse/client/query/QueryServerHttpCompressionTests.java index 57e9cad16..555115914 100644 --- a/client-v2/src/test/java/com/clickhouse/client/query/QueryServerHttpCompressionTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/query/QueryServerHttpCompressionTests.java @@ -1,6 +1,8 @@ package com.clickhouse.client.query; +import com.clickhouse.client.api.Client; import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader; +import com.clickhouse.client.api.enums.CompressionAlgorithm; import com.clickhouse.client.api.query.GenericRecord; import com.clickhouse.client.api.query.QuerySettings; import org.apache.hc.core5.http.HttpHeaders; @@ -47,6 +49,29 @@ public void testQueryCompressed(String compressAlgo) throws Exception { } } + @Test(groups = {"integration"}, dataProvider = "compressionAlgorithms") + public void testQueryWithCompressionAlgorithmAndHttpCompression(CompressionAlgorithm algorithm) throws Exception { + try (Client client = newClient() + .compressionAlgorithm(algorithm) + .compressClientRequest(true) + .build()) { + List records = client.queryAll("SELECT number, toString(number) AS str " + + "FROM system.numbers LIMIT 1000"); + + Assert.assertEquals(records.size(), 1000); + Assert.assertEquals(records.get(999).getLong("number"), 999); + Assert.assertEquals(records.get(999).getString("str"), "999"); + } + } + + @DataProvider(name = "compressionAlgorithms") + public Object[][] compressionAlgorithms() { + return new Object[][]{ + {CompressionAlgorithm.LZ4}, + {CompressionAlgorithm.ZSTD}, + }; + } + @DataProvider(name = "testQueryCompressedProvider") public Object[][] testQueryCompressedProvider() { return new Object[][] { diff --git a/docs/features.md b/docs/features.md index 69112495f..fa0bc582e 100644 --- a/docs/features.md +++ b/docs/features.md @@ -32,7 +32,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Session handling: Supports client-wide and per-operation HTTP sessions, operation-level session overrides, runtime updates of client `session_id`, and server-side session validation through `session_check`. - Metadata discovery: Loads table schemas from table names or queries and allows schema registration for typed read/write operations. - Server information loading: Can refresh server version, current user, and server time zone information. -- Compression support: Supports response compression, ClickHouse LZ4 request compression, HTTP content compression, and caller-supplied precompressed insert bodies. The algorithm of a compressed body is selected with `client.compression_algorithm` (`LZ4` by default, also `ZSTD`, `GZIP` and `NONE`) and is requested with the HTTP content coding of the operation, so a compressed response uses the algorithm the client asked for on every server version. +- Compression support: Supports response compression, ClickHouse LZ4 request compression, HTTP content compression, and caller-supplied precompressed insert bodies. The algorithm of a compressed body is selected with `client.compression_algorithm` (`LZ4` by default, also `ZSTD`) and is requested with the HTTP content coding of the operation, so a compressed response uses the algorithm the client asked for on every server version. The property selects only how a body is compressed; whether it is compressed stays with the `compress_server_response` and `compress_client_request` flags. - Retry behavior: Can retry failed operations for configured failure causes and retry limits. - Client-side request cancellation: `Client.cancelTransportRequest(String queryId)` aborts the in-flight HTTP request and its IO for the operation started with the given query id. It requires the caller to set the query id in operation settings, is best-effort (it cancels client-side IO but the result is not guaranteed), and does not issue a server-side `KILL QUERY` - the server stops the query on its own once the client disconnects. A cancelled operation that is being retried stops instead of issuing another request, also when the cancellation lands between two attempts (for example from `DataStreamWriter#onRetry()`). - Metrics and observability: Exposes client/server operation metrics and optionally integrates connection-pool gauges with Micrometer. diff --git a/docs/releases/0_11_0.md b/docs/releases/0_11_0.md index 31d4b0006..cae6ed295 100644 --- a/docs/releases/0_11_0.md +++ b/docs/releases/0_11_0.md @@ -13,19 +13,19 @@ The client now requests a response with the HTTP content coding of the algorithm a compressed body is always the one the client asked for: - The new property `client.compression_algorithm` (builder method `Client.Builder#compressionAlgorithm`) selects the - algorithm out of `LZ4` (default), `ZSTD`, `GZIP` and `NONE`. Both the name and the content-coding token of an - algorithm are accepted, in any case. + algorithm out of `LZ4` (default) and `ZSTD`. Both the name and the content-coding token of an algorithm are + accepted, in any case. - The default is `LZ4`, so an application that does not set the property keeps reading `LZ4` on every server version. -- `ZSTD` needs `com.github.luben:zstd-jni` on the classpath, which the client does not bring: the dependency of - `clickhouse-jdbc` stays `provided`, so packaging is unchanged and an application that selects `ZSTD` declares the - dependency itself. -- `NONE` disables compression of the request and the response, whatever `compress` and `decompress` are set to. +- `ZSTD` needs `com.github.luben:zstd-jni`, which the client now brings itself: the dependency of `clickhouse-jdbc` + is no longer `provided`, so the shaded jars carry it and an application needs no additional dependency. +- The property selects only *how* a body is compressed. *Whether* a body is compressed stays with + `Client.Builder#compressServerResponse` and `Client.Builder#compressClientRequest`. What to check in an application: - **A user profile that forbids setting changes.** A client that reads a compressed response now also sends `enable_http_compression=1`, which the server rejects for a profile with `readonly = 1`. Use `readonly = 2`, which - allows setting changes, or set `client.compression_algorithm` to `NONE`. + allows setting changes, or read an uncompressed response with `compressServerResponse(false)`. - **Code that inspects the response encoding.** A compressed response now carries `Content-Encoding` with the requested coding instead of the `compress=1` framing of ClickHouse.