From c4c78b9f451069c8dc94689c46c45a51d74074d6 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sat, 1 Aug 2026 19:26:47 +0900 Subject: [PATCH 1/2] fix(bigquerystorage): accept base64 and byte[] values for BYTES columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JsonToProtoMessage accepts only ByteString or a JSONArray of integer byte values for a BYTES column, so a base64 string fails per record — the encoding protobuf's canonical JSON mapping and BigQuery's own JSON ingestion both use. A scalar byte[] was also rejected while the repeated path accepted one. Decoding is guarded on the TableSchema saying BYTES, because proto BYTES also carries NUMERIC and BIGNUMERIC; without that guard a string on a NUMERIC column would silently decode instead of erroring. Standard and URL-safe alphabets are both accepted, with or without padding, as that mapping requires. Fixes #13980 Fixes #13979 --- .../storage/v1/JsonToProtoMessage.java | 37 ++++++ .../storage/v1/JsonToProtoMessageTest.java | 118 ++++++++++++++++++ 2 files changed, 155 insertions(+) diff --git a/java-bigquerystorage/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/java-bigquerystorage/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index 6e5643f00270..66be1a8df190 100644 --- a/java-bigquerystorage/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/java-bigquerystorage/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -48,6 +48,7 @@ import java.time.temporal.ChronoField; import java.time.temporal.TemporalAccessor; import java.util.ArrayList; +import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -624,6 +625,9 @@ private void fillField( if (val instanceof ByteString) { protoMsg.setField(fieldDescriptor, ((ByteString) val).toByteArray()); return; + } else if (val instanceof byte[]) { + protoMsg.setField(fieldDescriptor, val); + return; } else if (val instanceof JSONArray) { byte[] bytes = new byte[((JSONArray) val).length()]; for (int j = 0; j < ((JSONArray) val).length(); j++) { @@ -636,6 +640,11 @@ private void fillField( } protoMsg.setField(fieldDescriptor, bytes); return; + } else if (val instanceof String + && fieldSchema != null + && fieldSchema.getType() == TableFieldSchema.Type.BYTES) { + protoMsg.setField(fieldDescriptor, parseBase64(currentScope, (String) val)); + return; } break; case INT64: @@ -901,6 +910,11 @@ private void fillRepeatedField( + index + "] could not be converted to byte[].")); } + } else if (val instanceof String + && fieldSchema != null + && fieldSchema.getType() == TableFieldSchema.Type.BYTES) { + protoMsg.addRepeatedField( + fieldDescriptor, parseBase64(currentScope + "[" + index + "]", (String) val)); } else { throwWrongFieldType(fieldDescriptor, currentScope, index); } @@ -1105,6 +1119,29 @@ private static void throwWrongFieldType( FIELD_TYPE_TO_DEBUG_MESSAGE.get(fieldDescriptor.getType()), currentScope, index)); } + /** + * Decodes a base64 string into bytes, the encoding protobuf's canonical JSON mapping defines for + * the {@code bytes} type. Standard and URL-safe alphabets are both accepted, with or without + * padding, as that mapping requires; the two alphabets are mutually exclusive on {@code +/} + * versus {@code -_}, so the fallback is what makes both work. + * + * @throws IllegalArgumentException if the value decodes under neither alphabet + */ + private static byte[] parseBase64(String currentScope, String value) { + try { + return Base64.getDecoder().decode(value); + } catch (IllegalArgumentException standardAlphabetFailed) { + try { + return Base64.getUrlDecoder().decode(value); + } catch (IllegalArgumentException urlSafeAlphabetFailed) { + throw new IllegalArgumentException( + "Error: " + + currentScope + + " could not be converted to byte[]: not a valid base64 string."); + } + } + } + /** * Internal helper method to check that the timestamp follows the expected String input of ISO8601 * string. Allows the fractional portion of the timestamp to support up to 12 digits of precision diff --git a/java-bigquerystorage/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java b/java-bigquerystorage/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java index bc5447215933..ea64d3bc66fa 100644 --- a/java-bigquerystorage/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java +++ b/java-bigquerystorage/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java @@ -1112,6 +1112,124 @@ void testDate() throws Exception { assertEquals(expectedProto, protoMsg); } + private TableSchema bytesTableSchema(String fieldName, TableFieldSchema.Mode mode) { + return TableSchema.newBuilder() + .addFields(TableFieldSchema.newBuilder(TEST_BYTES).setName(fieldName).setMode(mode).build()) + .build(); + } + + @Test + void testBytesFromBase64() throws Exception { + TableSchema tableSchema = bytesTableSchema("test_field_type", TableFieldSchema.Mode.NULLABLE); + BytesType expectedProto = + BytesType.newBuilder().setTestFieldType(ByteString.copyFromUtf8("hi")).build(); + // Base64 for "hi". Padding is optional in protobuf's canonical JSON mapping, and "hi" encodes + // to characters the standard and URL-safe alphabets share. + for (String encoded : ImmutableList.of("aGk=", "aGk")) { + JSONObject json = new JSONObject(); + json.put("test_field_type", encoded); + DynamicMessage protoMsg = + JsonToProtoMessage.INSTANCE.convertToProtoMessage( + BytesType.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + } + + @Test + void testBytesFromBase64EitherAlphabet() throws Exception { + TableSchema tableSchema = bytesTableSchema("test_field_type", TableFieldSchema.Mode.NULLABLE); + BytesType expectedProto = + BytesType.newBuilder().setTestFieldType(ByteString.copyFrom(new byte[] {-5, -1})).build(); + // 0xFB 0xFF is "+/8" in the standard alphabet and "-_8" in the URL-safe one — the only place + // the two differ. Protobuf's mapping accepts both, with or without padding. + for (String encoded : ImmutableList.of("+/8=", "-_8=", "+/8", "-_8")) { + JSONObject json = new JSONObject(); + json.put("test_field_type", encoded); + DynamicMessage protoMsg = + JsonToProtoMessage.INSTANCE.convertToProtoMessage( + BytesType.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + } + + @Test + void testBytesRepeatedFromBase64() throws Exception { + TableSchema tableSchema = bytesTableSchema("test_repeated", TableFieldSchema.Mode.REPEATED); + RepeatedBytes expectedProto = + RepeatedBytes.newBuilder() + .addTestRepeated(ByteString.copyFromUtf8("hi")) + .addTestRepeated(ByteString.copyFromUtf8("yo")) + .build(); + JSONObject json = new JSONObject(); + json.put("test_repeated", new JSONArray(ImmutableList.of("aGk=", "eW8="))); + DynamicMessage protoMsg = + JsonToProtoMessage.INSTANCE.convertToProtoMessage( + RepeatedBytes.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + + @Test + void testBytesFromInvalidBase64() throws Exception { + TableSchema tableSchema = bytesTableSchema("test_field_type", TableFieldSchema.Mode.NULLABLE); + JSONObject json = new JSONObject(); + json.put("test_field_type", "not base64 at all"); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + JsonToProtoMessage.INSTANCE.convertToProtoMessage( + BytesType.getDescriptor(), tableSchema, json)); + assertTrue( + e.getMessage().contains("could not be converted to byte[]: not a valid base64 string.")); + } + + @Test + void testBytesFromBase64NeedsTableSchema() throws Exception { + // Without a TableSchema the converter cannot tell BYTES from NUMERIC and BIGNUMERIC, which + // share the same proto type, so a string stays an error rather than being decoded. + JSONObject json = new JSONObject(); + json.put("test_field_type", "aGk="); + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + JsonToProtoMessage.INSTANCE.convertToProtoMessage(BytesType.getDescriptor(), json)); + assertTrue( + e.getMessage().contains("JSONObject does not have a bytes field at root.test_field_type.")); + } + + @Test + void testNumericFromStringIsNotDecodedAsBase64() throws Exception { + TableSchema tableSchema = + TableSchema.newBuilder() + .addFields(TableFieldSchema.newBuilder(TEST_NUMERIC).setName("test_field_type").build()) + .build(); + BytesType expectedProto = + BytesType.newBuilder() + .setTestFieldType( + BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.5"))) + .build(); + JSONObject json = new JSONObject(); + json.put("test_field_type", "1.5"); + DynamicMessage protoMsg = + JsonToProtoMessage.INSTANCE.convertToProtoMessage( + BytesType.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + + @Test + void testBytesFromByteArray() throws Exception { + TableSchema tableSchema = bytesTableSchema("test_field_type", TableFieldSchema.Mode.NULLABLE); + BytesType expectedProto = + BytesType.newBuilder().setTestFieldType(ByteString.copyFromUtf8("hi")).build(); + JSONObject json = new JSONObject(); + json.put("test_field_type", new byte[] {104, 105}); + DynamicMessage protoMsg = + JsonToProtoMessage.INSTANCE.convertToProtoMessage( + BytesType.getDescriptor(), tableSchema, json); + assertEquals(expectedProto, protoMsg); + } + @Test void testAllTypes() throws Exception { for (Map.Entry entry : AllTypesToDebugMessageTest.entrySet()) { From 931c717f6f3b8ce925066ba1044531ee968a47e5 Mon Sep 17 00:00:00 2001 From: laughingman7743 Date: Sat, 1 Aug 2026 19:39:18 +0900 Subject: [PATCH 2/2] fix(bigquerystorage): pick the base64 alphabet up front instead of by fallback Falling back through a caught IllegalArgumentException put an exception on the happy path for every record when a producer uses the URL-safe alphabet. A value containing - or _ is never valid standard base64, so the alphabet can be chosen before decoding. The original exception is now kept as the cause. --- .../storage/v1/JsonToProtoMessage.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/java-bigquerystorage/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java b/java-bigquerystorage/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java index 66be1a8df190..b5e90d027b33 100644 --- a/java-bigquerystorage/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java +++ b/java-bigquerystorage/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java @@ -1122,23 +1122,24 @@ private static void throwWrongFieldType( /** * Decodes a base64 string into bytes, the encoding protobuf's canonical JSON mapping defines for * the {@code bytes} type. Standard and URL-safe alphabets are both accepted, with or without - * padding, as that mapping requires; the two alphabets are mutually exclusive on {@code +/} - * versus {@code -_}, so the fallback is what makes both work. + * padding, as that mapping requires. The two differ only in {@code +/} versus {@code -_}, so the + * alphabet is picked by looking for the URL-safe characters rather than by decoding twice, which + * would make an exception part of the happy path for every URL-safe record. * - * @throws IllegalArgumentException if the value decodes under neither alphabet + * @throws IllegalArgumentException if the value is not valid base64 */ private static byte[] parseBase64(String currentScope, String value) { try { - return Base64.getDecoder().decode(value); - } catch (IllegalArgumentException standardAlphabetFailed) { - try { + if (value.indexOf('-') >= 0 || value.indexOf('_') >= 0) { return Base64.getUrlDecoder().decode(value); - } catch (IllegalArgumentException urlSafeAlphabetFailed) { - throw new IllegalArgumentException( - "Error: " - + currentScope - + " could not be converted to byte[]: not a valid base64 string."); } + return Base64.getDecoder().decode(value); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Error: " + + currentScope + + " could not be converted to byte[]: not a valid base64 string.", + e); } }