Skip to content
Merged
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,15 @@
and the comma-separated `ssl_cipher_suites` connection property (client-v2 and jdbc-v2) restrict the cipher suites
enabled on secure connections; when unset, the transport defaults are used. Cipher-suite selection is independent of the
trust configuration and `ssl_mode`. (https://github.com/ClickHouse/clickhouse-java/issues/2882)
- **[client-v2, jdbc-v2]** Added support for un-flattened `Nested(...)` columns (tables created with
`flatten_nested = 0`). Previously the `RowBinary` writer threw `UnsupportedOperationException: Unsupported
data type: Nested` when inserting such a column. `client-v2` now serializes a `Nested(f1 T1, ..., fN TN)`
column the same way it is read — identically to `Array(Tuple(T1, ..., TN))` (a var-uint row count followed
by one tuple per nested row) — so it can be written through the insert path / `RowBinaryFormatWriter`. In
`jdbc-v2` an un-flattened `Nested(...)` column is exposed as a JDBC `ARRAY` whose element type is
`Tuple(f1 T1, ..., fN TN)`: it can be inserted through `Connection#createArrayOf`/`setArray` or `setObject`
and read back through `getArray`/`getObject`, and `java.sql.Array#getResultSet()` iterates the nested rows
as `(INDEX, VALUE)` pairs where each `VALUE` is the tuple. (https://github.com/ClickHouse/clickhouse-java/issues/2477)
- **[client-v2, jdbc-v2]** Added logging on previously-silent error and diagnostic paths (no functional or
public-API change). (https://github.com/ClickHouse/clickhouse-java/issues/2969)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo
case Map:
serializeMapData(stream, value, column);
break;
case Nested:
serializeNestedTypeData(stream, value, column);
break;
case SimpleAggregateFunction:
// A SimpleAggregateFunction(func, T) value serializes identically to its underlying
// type T. This is a deliberate exception to serializeNestedData's "nested elements
Expand Down Expand Up @@ -146,6 +149,25 @@ private static void serializeNestedData(OutputStream stream, Object value, Click
serializeData(stream, value, column);
}

/**
* Serializes a {@code Nested} column. In {@code RowBinary} a {@code Nested(f1 T1, ..., fN TN)}
* column has the same layout as {@code Array(Tuple(T1, ..., TN))}: a var-uint element count
* followed by that many tuples, each carrying the N field values in declaration order. The
* value is therefore a list (or array) of tuples, matching what
* {@link BinaryStreamReader#readNested(ClickHouseColumn)} produces on read.
*/
private static void serializeNestedTypeData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
Comment thread
chernser marked this conversation as resolved.
if (value == null) {
writeVarInt(stream, 0);
return;
Comment thread
cursor[bot] marked this conversation as resolved.
}
List<?> tuples = convertArrayValueToList(value);
writeVarInt(stream, tuples.size());
for (Object tuple : tuples) {
serializeTupleData(stream, tuple, column);
}
Comment thread
cursor[bot] marked this conversation as resolved.
}

private static final Map<Class<?>, ClickHouseColumn> PREDEFINED_TYPE_COLUMNS = getPredefinedTypeColumnsMap();

private static Map<Class<?>, ClickHouseColumn> getPredefinedTypeColumnsMap() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -387,7 +387,7 @@
}

@Test(dataProvider = "nestedNullableData")
public void testNestedNullableRoundTrip(String typeName, Object value) throws Exception {

Check warning on line 390 in client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Replace these 3 tests with a single Parameterized one.

See more on https://sonarcloud.io/project/issues?id=ClickHouse_clickhouse-java&issues=AZ-sqFuJ1e_TMkRk1E6I&open=AZ-sqFuJ1e_TMkRk1E6I&pullRequest=2936
ClickHouseColumn column = ClickHouseColumn.of("v", typeName);

ByteArrayOutputStream out = new ByteArrayOutputStream();
Expand Down Expand Up @@ -460,6 +460,17 @@
};
}

@Test(dataProvider = "rowBinaryTypeData")
public void testRowBinaryTypeRoundTrip(String typeName, Object value) throws Exception {

Check warning on line 464 in client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/SerializerUtilsTest.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Update this method so that its implementation is not identical to "testNestedNullableRoundTrip" on line 390.

See more on https://sonarcloud.io/project/issues?id=ClickHouse_clickhouse-java&issues=AZ9yUVxyFoG4PDw31zWH&open=AZ9yUVxyFoG4PDw31zWH&pullRequest=2936
ClickHouseColumn column = ClickHouseColumn.of("v", typeName);

ByteArrayOutputStream out = new ByteArrayOutputStream();
SerializerUtils.serializeData(out, value, column);

Object actual = newReader(out.toByteArray()).readValue(column);
Assert.assertEquals(normalize(actual), normalize(value));
}

@Test(dataProvider = "simpleAggregateFunctionData")
public void testSimpleAggregateFunctionRoundTrip(String typeName, Object value) throws Exception {
ClickHouseColumn column = ClickHouseColumn.of("v", typeName);
Expand All @@ -471,6 +482,33 @@
Assert.assertEquals(normalize(actual), normalize(value));
}

@DataProvider(name = "rowBinaryTypeData")
private Object[][] rowBinaryTypeData() {
return new Object[][] {
// A Nested(...) column has the same RowBinary layout as Array(Tuple(...)): a
// var-uint row count followed by that many tuples. Two rows detect a wrong count
// or a dropped field byte, which would shift every following tuple.
{"Nested(a Int32, b String)",
Arrays.asList(Arrays.asList(1, "x"), Arrays.asList(2, "y"))},

// Same value as arrays instead of Lists: an Object[][] of Object[] rows (a
// "matrix" array). convertArrayValueToList takes the array branch and each row is
// serialized by serializeTupleData's array branch, producing the same bytes as the
// List-shaped case above.
{"Nested(a Int32, b String)",
new Object[][] {{1, "x"}, {2, "y"}}},

// A Nullable field in the MIDDLE of the nested tuple, with a trailing fixed-width
// Float64: a dropped null-marker byte misaligns the Float64 and is caught. The
// second row exercises the null branch of that field.
{"Nested(a Int32, b Nullable(String), c Float64)",
Arrays.asList(Arrays.asList(7, "opt", 9.5d), Arrays.asList(7, null, 8.5d))},

// An empty Nested serializes as a zero-length array.
{"Nested(a Int32, b String)", Arrays.asList()},
};
}

@DataProvider(name = "simpleAggregateFunctionData")
private Object[][] simpleAggregateFunctionData() {
return new Object[][] {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -762,6 +762,82 @@
writeTest(tableName, tableCreate, rows);
}

@Test (groups = { "integration" })
public void writeNestedTypeTests() throws Exception {
String tableName = "rowBinaryFormatWriterTest_writeNestedTypeTests_" + UUID.randomUUID().toString().replace('-', '_');
// flatten_nested = 0 keeps the column typed as Nested(...) in the schema instead of
// expanding it into parallel Array(...) sub-columns; the un-flattened Nested type is the
// one that has to be serialized as Array(Tuple(...)).
String tableCreate = "CREATE TABLE \"" + tableName + "\" " +
" (id Int32, " +
" n Nested(a UInt32, b Nullable(String)) " +
" ) Engine = MergeTree ORDER BY id SETTINGS flatten_nested = 0";

List<Object> nested = Arrays.asList(Arrays.asList(10L, "x"), Arrays.asList(20L, null));
Field[][] rows = new Field[][] {{
new Field("id", 1), //Row ID
new Field("n", nested).set(nested) //Nested
}
};

writeTest(tableName, tableCreate, rows);
}

@Test (groups = { "integration" })
public void writeNestedWithArrayAndComplexFieldsTests() throws Exception {
String tableName = "rowBinaryFormatWriterTest_writeNestedComplex_" + UUID.randomUUID().toString().replace('-', '_');
// Un-flattened Nested whose fields cover arrays, nested arrays, an inner tuple and a
// nullable, with a fixed-width Float64 last so a dropped/extra byte shifts the trailing
// field and is detected. Serialized as Array(Tuple(...)) per nested row.
String tableCreate = "CREATE TABLE \"" + tableName + "\" " +
" (id Int32, " +
" n Nested(a UInt32, tags Array(String), matrix Array(Array(Int32)), " +
" pair Tuple(Int8, String), score Nullable(Float64), tail Float64) " +
" ) Engine = MergeTree ORDER BY id SETTINGS flatten_nested = 0";

List<Object> nestedMulti = Arrays.asList(
Arrays.asList(10L, Arrays.asList("x", "y"), Arrays.asList(Arrays.asList(1, 2), Arrays.asList(3)),
Arrays.asList(1, "p"), 1.5, 9.5),
Arrays.asList(20L, Collections.emptyList(), Arrays.asList(Arrays.asList(4, 5)),
Arrays.asList(2, "q"), null, 8.5));
List<Object> nestedEmpty = Collections.emptyList();
List<Object> nestedSingle = Arrays.asList(
Arrays.asList(30L, Arrays.asList("z"), Arrays.asList(Arrays.asList(7, 8, 9)),
Arrays.asList(3, "r"), 2.5, 7.5));

Field[][] rows = new Field[][] {
{new Field("id", 1), new Field("n", nestedMulti)},
{new Field("id", 2), new Field("n", nestedEmpty)},
{new Field("id", 3), new Field("n", nestedSingle)},
};

writeTest(tableName, tableCreate, rows);
}

@Test (groups = { "integration" })
public void writeNestedWithVariedScalarTypesTests() throws Exception {
String tableName = "rowBinaryFormatWriterTest_writeNestedScalars_" + UUID.randomUUID().toString().replace('-', '_');
// Un-flattened Nested spanning several scalar type families (unsigned, temporal, UUID,
// low-cardinality, boolean) with a fixed-width Float64 last.
String tableCreate = "CREATE TABLE \"" + tableName + "\" " +
" (id Int32, " +
" n Nested(u UInt64, when Date, guid UUID, lc LowCardinality(String), flag Bool, tail Float64) " +
" ) Engine = MergeTree ORDER BY id SETTINGS flatten_nested = 0";

UUID g1 = UUID.fromString("00112233-4455-6677-8899-aabbccddeeff");
UUID g2 = UUID.fromString("ffeeddcc-bbaa-9988-7766-554433221100");
List<Object> nested = Arrays.asList(
Arrays.asList(123456789L, LocalDate.of(2023, 1, 2), g1, "cat", true, 1.25),

Check warning on line 830 in client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Use a "java.time.Month" enum constant instead of this int literal.

See more on https://sonarcloud.io/project/issues?id=ClickHouse_clickhouse-java&issues=AZ-qNx6_EVOTEArumSQ5&open=AZ-qNx6_EVOTEArumSQ5&pullRequest=2936
Arrays.asList(0L, LocalDate.of(1970, 1, 1), g2, "dog", false, 2.5));

Check warning on line 831 in client-v2/src/test/java/com/clickhouse/client/datatypes/RowBinaryFormatWriterTest.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Use a "java.time.Month" enum constant instead of this int literal.

See more on https://sonarcloud.io/project/issues?id=ClickHouse_clickhouse-java&issues=AZ-qNx6_EVOTEArumSQ6&open=AZ-qNx6_EVOTEArumSQ6&pullRequest=2936

Field[][] rows = new Field[][] {{
new Field("id", 1), //Row ID
new Field("n", nested) //Nested
}};

writeTest(tableName, tableCreate, rows);
}

@Test (groups = { "integration" })
public void writeNullableTests() throws Exception {
String tableName = "rowBinaryFormatWriterTest_writeNullableTests_" + UUID.randomUUID().toString().replace('-', '_');
Expand Down
2 changes: 2 additions & 0 deletions docs/features.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t
- BFloat16 type support: For ClickHouse `24.11+`, reads and writes the `BFloat16` type through generic records, binary readers, POJO binding, and `Nullable`/`Dynamic`/`Variant` wrappers. `BFloat16` maps to the Java `float` type: a read widens the stored 16-bit value losslessly, while a write keeps the high 16 bits of the `float`. In `jdbc-v2`, `BFloat16` maps to `java.sql.Types.FLOAT` / `java.lang.Float` and is read and written through the standard `getFloat`/`setFloat` and `getObject` accessors.
- QBit type support: For ClickHouse `25.10+` (requires the `allow_experimental_qbit_type` server setting to create a column), reads and writes the experimental `QBit(element_type, dimension[, stride])` vector type. The type-name parser accepts two or three parameters (the optional third parameter is the stride) and recognizes the documented element types `Int8`, `BFloat16`, `Float32`, and `Float64`; an element type outside that documented set is parsed with a warning rather than rejected, so a newer server-side element type does not require a client change to parse. On the wire a `QBit` value is encoded exactly like `Array(element_type)` (a length-prefixed list of elements), so the client reads and writes it as a Java array of the element type — `float[]` for `BFloat16`/`Float32`, `double[]` for `Float64` — through generic records, binary readers, and POJO binding, using a dedicated `QBit` read/serialize path (the shared `Array`-like wire encoding is an implementation detail, not a type equivalence). A `QBit` held inside a `Dynamic`/`Variant`/`JSON` column is decoded on read: its binary type encoding (`0x36 <element_type> <dimension>`) is read back to the concrete `QBit(...)` type. The client never infers a `QBit` from a Java value, so writing a `QBit` into a `Dynamic` column is not supported and is rejected with a clear `ClientException` rather than emitting an incomplete tag that would desynchronize the stream. This `Array`-like encoding is what the server uses over `RowBinary` formats; the `Native` format instead transmits `QBit` using its internal bit-transposed `Tuple(FixedString(...))` layout, which the client does not decode, so reading any column that is or contains a `QBit` (including a nested `QBit`, e.g. `Map(String, QBit(...))`) through the `Native` format is rejected with a clear `ClientException` (use a `RowBinary` format such as `RowBinaryWithNamesAndTypes` instead).
- Geometry type support: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, the client reads and writes `Geometry` values through generic records, binary readers, POJO binding, and SQL parameter formatting, using Java array dimensionality to represent the geometry shape.
- Nested type support: Un-flattened `Nested(f1 T1, ..., fN TN)` columns (tables created with `flatten_nested = 0`) can be written through the insert path (`Client#insert`) using `RowBinaryFormatWriter#setValue`, and are read back through the binary readers and generic records. The column is serialized the same way it is read — identically to `Array(Tuple(T1, ..., TN))`, a var-uint row count followed by one tuple per nested row — so the value supplied for the column is a `List` (or array) of tuples, one tuple per nested row, each carrying the N field values in declaration order.
- Insert APIs: Supports inserting registered POJOs, raw streams, and callback-driven writers, with optional column lists and format selection.
- Insert controls: Supports insert-specific settings such as deduplication token, query id, compression behavior, and request headers.
- Command execution: Executes DDL or other non-result commands and exposes response summaries and operation metrics.
Expand Down Expand Up @@ -85,6 +86,7 @@ Compatibility-sensitive traits:
- QBit type mapping: For ClickHouse `25.10+`, JDBC exposes the experimental `QBit(element_type, dimension)` type as `ARRAY`, returning the vector as a `java.sql.Array` of the element type from `getObject()`/`getArray()`. Supported element types are `BFloat16` and `Float32` (both `java.lang.Float`) and `Float64` (`java.lang.Double`). The `allow_experimental_qbit_type` server setting is required to create a `QBit` column.
- Custom result-set type map: `ResultSet#getObject(int|String, Map<String, Class<?>>)` accepts both ClickHouse type names and JDBC `SQLType` names as map keys. Only unwrapped type names are used — `Nullable(...)` and `LowCardinality(...)` wrappers are stripped before lookup, so a key like `"Int32"` matches both `Int32` and `Nullable(Int32)` columns, and keys like `"Nullable(Int32)"` are not recognized. Lookup order is `ClickHouseColumn#getDataType().name()` (e.g. `"Int32"`, `"String"`, `"DateTime"`) then `SQLType.getName()` (e.g. `"INTEGER"`, `"VARCHAR"`, `"TIMESTAMP"`); a missing entry leaves the value uncoerced (read as-is). The feature is supported for primitive ClickHouse types only — `Array`, `Tuple`, `Map`, `Nested`, and geometry types (`Point`, `Ring`, `LineString`, `Polygon`, `MultiPolygon`, `MultiLineString`, `Geometry`) bypass the type map and are returned in their native form.
- Arrays and tuples: Supports JDBC arrays plus ClickHouse tuple values through custom `Array` and `Struct` implementations.
- Nested columns: Un-flattened `Nested(f1 T1, ..., fN TN)` columns (tables created with `flatten_nested = 0`) are exposed as JDBC `ARRAY` whose element type is `Tuple(f1 T1, ..., fN TN)`. They can be inserted through `Connection#createArrayOf`/`setArray` or `setObject` (a Java array of tuples) and read back through `getArray`/`getObject`; `java.sql.Array#getResultSet()` iterates the nested rows as `(INDEX, VALUE)` pairs where each `VALUE` is the tuple.
Comment thread
polyglotAI-bot marked this conversation as resolved.
- Geometry type mapping: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, JDBC exposes `Geometry` as `ARRAY`, returns nested Java arrays from `getObject()`/`getArray()`, and accepts `Struct` or nested `Array` inputs for prepared-statement inserts depending on the geometry shape.
- Client info propagation: Supports JDBC client info such as `ApplicationName` and forwards it to the underlying client name.
- Wrapper support: Implements standard JDBC `Wrapper` and `unwrap` behavior on major JDBC objects.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,7 @@ public ArrayResultSet(Object array, ClickHouseColumn column) {
this.column = column;
this.columnCount = 2; // INDEX, VALUE

List<ClickHouseColumn> nestedColumns = column.getNestedColumns();
ClickHouseColumn valueColumn = column.getArrayNestedLevel() == 1 ? column.getArrayBaseColumn() : nestedColumns.get(0);
ClickHouseColumn valueColumn = elementColumn(column);
this.metadata = new ResultSetMetaDataImpl(Arrays.asList(INDEX_COLUMN, ClickHouseColumn.parse(VALUE_COLUMN + " "
+ valueColumn.getOriginalTypeName()).get(0))
, "", "", "", JdbcUtils.DATA_TYPE_CLASS_MAP, java.util.Collections.emptyMap());
Expand All @@ -74,6 +73,29 @@ public ArrayResultSet(Object array, ClickHouseColumn column) {
indexConverterMap = defaultValueConverters.getConvertersForType(Integer.class);
}

/**
* Resolves the type of a single element of {@code column} (an array-like column). A
* {@code Nested(f1 T1, ..., fN TN)} column is read as an array whose elements are
* {@code Tuple(f1 T1, ..., fN TN)}, so its element is that tuple rather than the first nested
* field; every other array-like column keeps its existing base/nested-column resolution.
*/
private static ClickHouseColumn elementColumn(ClickHouseColumn column) {
List<ClickHouseColumn> nestedColumns = column.getNestedColumns();
if (column.isNested()) {
StringBuilder tupleType = new StringBuilder(ClickHouseDataType.Tuple.name()).append('(');
for (int i = 0; i < nestedColumns.size(); i++) {
ClickHouseColumn field = nestedColumns.get(i);
if (i > 0) {
tupleType.append(", ");
}
tupleType.append(field.getColumnName()).append(' ').append(field.getOriginalTypeName());
}
tupleType.append(')');
return ClickHouseColumn.of(VALUE_COLUMN, tupleType.toString());
}
return column.getArrayNestedLevel() == 1 ? column.getArrayBaseColumn() : nestedColumns.get(0);
}
Comment thread
cursor[bot] marked this conversation as resolved.

@Override
public boolean next() throws SQLException {
if (pos + 1 >= length || length == 0) {
Expand Down Expand Up @@ -139,9 +161,7 @@ private Object getValueAsObject(int columnIndex, Class<?> type, Object defaultVa
}

if (type == Array.class) {
ClickHouseColumn nestedColumn =
column.getArrayNestedLevel() == 1 ? column.getArrayBaseColumn() : column.getNestedColumns().get(0);
return new com.clickhouse.jdbc.types.Array(nestedColumn, JdbcUtils.arrayToObjectArray(value));
return new com.clickhouse.jdbc.types.Array(elementColumn(column), JdbcUtils.arrayToObjectArray(value));
} else {
Map<Class<?>, Function<Object, Object>> valueConverterMap = initValueConverterMapIfNeeded(value);
return convertValue(value, type, valueConverterMap);
Expand Down
Loading
Loading