From fa64920dbe839adab337e4ad81efe124e87171f6 Mon Sep 17 00:00:00 2001 From: jkaliak Date: Sun, 26 Jul 2026 16:48:05 +0200 Subject: [PATCH 1/3] Add failing tests for non-BMP text round-tripping (#25) StringUtilities::ToUtf8/FromUtf8 use std::codecvt_utf8, which converts UTF-8 to and from UCS-2 when wchar_t is 16 bits. Code points above U+FFFF therefore do not round-trip on Windows, while the same code is correct on Linux and macOS where wchar_t is 32 bits. These tests reproduce that: they are expected to fail on Windows and pass on Linux and macOS. That asymmetry is the signature of the defect - if the tests were to fail everywhere they would be measuring something else, and if they were to pass everywhere they would not reproduce the bug at all. Every wide literal uses universal-character escapes instead of literal non-ASCII source bytes. MSVC is not passed /utf-8, so it decodes such bytes in the system codepage; the pre-existing Greek literals in database_test.cc compare a mangled literal against the same mangled literal and so pass without testing anything. Expected UTF-8 is asserted as exact bytes, and the end-to-end test matches on hex(name) in SQL, so a conversion that is wrong in both directions cannot pass. Tests reach src/internal/string_utilities.h via a new include directory, so the conversion can be exercised directly and not only through Save/Fetch. Co-Authored-By: Claude Opus 5 --- tests/CMakeLists.txt | 4 ++ tests/database_test.cc | 23 ++++++++ tests/utf8_test.cc | 125 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 tests/utf8_test.cc diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c9cdce3..0b724d6 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,6 +8,10 @@ include(GoogleTest) include_directories ("${PROJECT_SOURCE_DIR}/include") include_directories ("./include") +# Internal headers (src/internal/...) so the UTF-8 conversion helpers can be tested directly +# rather than only through the public wstring API +include_directories ("${PROJECT_SOURCE_DIR}/src") + # Collect test sources into the variable TEST_SOURCES file (GLOB TEST_SOURCES "*.cpp" diff --git a/tests/database_test.cc b/tests/database_test.cc index cd35c26..6dd1f8f 100644 --- a/tests/database_test.cc +++ b/tests/database_test.cc @@ -874,3 +874,26 @@ TEST_F(DatabaseTest, RawSqlQueryForPersistedRecord) { EXPECT_EQ(52, fetched_persons[0].id); EXPECT_EQ(L"johnie", fetched_persons[0].first_name); } + +TEST_F(DatabaseTest, TextOutsideTheBasicMultilingualPlaneSurvivesSaveAndFetch) { + const auto db = Database::Instance(); + + // Code points above U+FFFF need a surrogate pair where wchar_t is 16 bits (Windows) and a + // single code unit where it is 32 bits (Linux, macOS). The stored UTF-8 and the fetched + // wstring must be identical on every platform. Written with universal-character escapes + // so the test does not depend on how the compiler decodes this file's source bytes. + const std::wstring name = L"\U0001F600"; + const std::wstring address = L"\U00010000\U0010FFFF"; + + db->Save(Company{name, 30, address, 50000.0, 1}); + + const auto fetched = db->Fetch(1); + EXPECT_EQ(name, fetched.name); + EXPECT_EQ(address, fetched.address); + + // The comparisons above are self-consistent: a conversion wrong in both directions would + // satisfy them. Match on the stored bytes instead - U+1F600 is F0 9F 98 80 in UTF-8 - so + // the row only disappears if what SQLite actually holds is correct UTF-8. + db->UnsafeSql("DELETE FROM Company WHERE hex(name) = 'F09F9880'"); + EXPECT_EQ(0, db->FetchAll().size()); +} diff --git a/tests/utf8_test.cc b/tests/utf8_test.cc new file mode 100644 index 0000000..a390953 --- /dev/null +++ b/tests/utf8_test.cc @@ -0,0 +1,125 @@ +// MIT License +// +// Copyright (c) 2026 Ioannis Kaliakatsos +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include + +#include + +#include "internal/string_utilities.h" + +using namespace sqlite_reflection; + +// Every wide literal in this file is written with universal-character escapes rather than +// literal non-ASCII source bytes. MSVC is not passed /utf-8, so it decodes non-ASCII source +// bytes in the system codepage; a test written with literal characters would compare a +// mangled literal against the same mangled literal and pass while proving nothing. Escapes +// are charset-independent, and the expected UTF-8 is asserted as exact bytes rather than by +// round-tripping, so a conversion that is wrong in both directions cannot pass either. + +namespace { +// A code point is encoded as a surrogate pair in a 16-bit wchar_t (Windows) and as a single +// code unit in a 32-bit wchar_t (Linux, macOS). The expected UTF-8 is identical on both. +const char* const kEmojiUtf8 = "\xF0\x9F\x98\x80"; // U+1F600 GRINNING FACE +const char* const kFirstSupplementaryUtf8 = "\xF0\x90\x80\x80"; // U+10000, first non-BMP +const char* const kLastCodePointUtf8 = "\xF4\x8F\xBF\xBF"; // U+10FFFF, highest valid +const char* const kLastBmpUtf8 = "\xEF\xBF\xBF"; // U+FFFF, last BMP code point + +std::wstring FromUtf8(const std::string& utf8) { + return StringUtilities::FromUtf8(utf8.data(), utf8.size()); +} +} // namespace + +TEST(Utf8Test, EncodesAsciiToExactBytes) { + EXPECT_EQ(std::string("Appleseed"), StringUtilities::ToUtf8(L"Appleseed")); +} + +TEST(Utf8Test, EncodesEmptyString) { + EXPECT_EQ(std::string(), StringUtilities::ToUtf8(std::wstring())); +} + +TEST(Utf8Test, EncodesBmpToExactBytes) { + // U+03C0 GREEK SMALL LETTER PI, U+03B1 GREEK SMALL LETTER ALPHA: two-byte sequences. + EXPECT_EQ(std::string("\xCF\x80\xCE\xB1"), StringUtilities::ToUtf8(L"\u03C0\u03B1")); +} + +TEST(Utf8Test, EncodesLastBmpCodePointToExactBytes) { + EXPECT_EQ(std::string(kLastBmpUtf8), StringUtilities::ToUtf8(L"\uFFFF")); +} + +TEST(Utf8Test, EncodesFirstSupplementaryCodePointToExactBytes) { + // U+10000 is the low side of the surrogate boundary: the first code point that needs a + // surrogate pair in UTF-16 and a four-byte UTF-8 sequence. + EXPECT_EQ(std::string(kFirstSupplementaryUtf8), StringUtilities::ToUtf8(L"\U00010000")); +} + +TEST(Utf8Test, EncodesEmojiToExactBytes) { + EXPECT_EQ(std::string(kEmojiUtf8), StringUtilities::ToUtf8(L"\U0001F600")); +} + +TEST(Utf8Test, EncodesLastValidCodePointToExactBytes) { + EXPECT_EQ(std::string(kLastCodePointUtf8), StringUtilities::ToUtf8(L"\U0010FFFF")); +} + +TEST(Utf8Test, EncodesSupplementaryCodePointMixedWithAscii) { + EXPECT_EQ(std::string("a") + kEmojiUtf8 + "b", StringUtilities::ToUtf8(L"a\U0001F600b")); +} + +TEST(Utf8Test, DecodesAscii) { + EXPECT_EQ(std::wstring(L"Appleseed"), FromUtf8("Appleseed")); +} + +TEST(Utf8Test, DecodesEmptyString) { + EXPECT_EQ(std::wstring(), FromUtf8(std::string())); +} + +TEST(Utf8Test, DecodesBmp) { + EXPECT_EQ(std::wstring(L"\u03C0\u03B1"), FromUtf8("\xCF\x80\xCE\xB1")); +} + +TEST(Utf8Test, DecodesLastBmpCodePoint) { + EXPECT_EQ(std::wstring(L"\uFFFF"), FromUtf8(kLastBmpUtf8)); +} + +TEST(Utf8Test, DecodesFirstSupplementaryCodePoint) { + EXPECT_EQ(std::wstring(L"\U00010000"), FromUtf8(kFirstSupplementaryUtf8)); +} + +TEST(Utf8Test, DecodesEmoji) { + EXPECT_EQ(std::wstring(L"\U0001F600"), FromUtf8(kEmojiUtf8)); +} + +TEST(Utf8Test, DecodesLastValidCodePoint) { + EXPECT_EQ(std::wstring(L"\U0010FFFF"), FromUtf8(kLastCodePointUtf8)); +} + +TEST(Utf8Test, RoundTripsSupplementaryCodePoints) { + const std::wstring original = L"\U0001F600\U00010000\U0010FFFF"; + EXPECT_EQ(original, FromUtf8(StringUtilities::ToUtf8(original))); +} + +TEST(Utf8Test, SupplementaryCodePointSurvivesLengthAndContent) { + // Guards against a conversion that silently drops or truncates the pair rather than + // producing a wrong-but-present result. + const std::wstring decoded = FromUtf8(kEmojiUtf8); + EXPECT_FALSE(decoded.empty()); + EXPECT_EQ(std::wstring(L"\U0001F600"), decoded); +} From e0f8f8fbc5794029a13480f9dbbf83bc2d5e1dd4 Mon Sep 17 00:00:00 2001 From: jkaliak Date: Sun, 26 Jul 2026 16:57:51 +0200 Subject: [PATCH 2/3] Fix non-BMP text round-tripping by replacing (#25) std::codecvt_utf8 transcodes one code unit per code point, so where wchar_t is 16 bits it mishandled everything above U+FFFF in both directions, and silently: - encoding emitted CESU-8, each surrogate half encoded as its own three-byte sequence (U+1F600 became ED A0 BD ED B8 80 instead of F0 9F 98 80); - decoding truncated the code point to its low 16 bits (U+1F600 became U+F600, and U+10000 became NUL). Nothing threw, so text was corrupted without any indication. The same code was correct where wchar_t is 32 bits, so identical data behaved differently per platform. is additionally deprecated since C++17 with no standard replacement, and this library's baseline is C++11. Conversion now pivots on Unicode code points in internal/utf8.h, and the wchar_t adapters in string_utilities.cc select the UTF-16 or UTF-32 path by the width of wchar_t. The public API is unchanged: text fields remain std::wstring. Malformed input is rejected with std::range_error rather than being altered, matching what std::wstring_convert did for ill-formed sequences. Rejected are overlong encodings, truncated sequences, invalid lead and continuation bytes, surrogates encoded directly in UTF-8, unpaired UTF-16 surrogates, and anything above U+10FFFF. The code-point core is tested directly, so the UTF-16 surrogate path - the one that was broken, and which is unreachable through the wstring API on Linux and macOS - is covered on every platform rather than only on Windows. Co-Authored-By: Claude Opus 5 --- README.md | 11 ++ src/internal/utf8.h | 63 +++++++++++ src/string_utilities.cc | 70 ++++++++++-- src/utf8.cc | 235 ++++++++++++++++++++++++++++++++++++++++ tests/utf8_test.cc | 119 ++++++++++++++++++++ 5 files changed, 488 insertions(+), 10 deletions(-) create mode 100644 src/internal/utf8.h create mode 100644 src/utf8.cc diff --git a/README.md b/README.md index 8ffad51..ec90b45 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,17 @@ Supported field macros: | `sqlite_reflection::TimePoint` | `MEMBER_DATETIME(name)` | | member function declaration | `FUNC(signature)` | +### Text encoding + +`MEMBER_TEXT` fields are `std::wstring` in C++ and are stored as UTF-8 in SQLite. The full +Unicode range is supported, including code points above U+FFFF such as emoji and the CJK +extension blocks, and text round-trips unchanged on every supported platform regardless of +whether `wchar_t` is 16 bits (Windows) or 32 bits (Linux, macOS). + +Text that is not valid Unicode is rejected rather than silently altered: converting a +`std::wstring` containing unpaired UTF-16 surrogates, or reading a column whose bytes are not +well-formed UTF-8, throws `std::range_error`. + ### Nullable fields Each field macro has a `_NULLABLE` variant (`MEMBER_INT_NULLABLE`, `MEMBER_REAL_NULLABLE`, diff --git a/src/internal/utf8.h b/src/internal/utf8.h new file mode 100644 index 0000000..f3f14ad --- /dev/null +++ b/src/internal/utf8.h @@ -0,0 +1,63 @@ +// MIT License +// +// Copyright (c) 2026 Ioannis Kaliakatsos +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once +#include +#include + +#include "reflection_export.h" + +namespace sqlite_reflection { +/// Unicode transcoding that does not depend on the platform width of wchar_t. +/// +/// The library previously converted text with std::wstring_convert and +/// std::codecvt_utf8. That facet transcodes UTF-8 to and from a single code unit per +/// code point, so where wchar_t is 16 bits (Windows) it silently mishandled everything above +/// U+FFFF: encoding produced CESU-8 (each surrogate half encoded separately, U+1F600 became +/// ED A0 BD ED B8 80 rather than F0 9F 98 80) and decoding truncated the code point to its low +/// 16 bits (U+1F600 became U+F600, U+10000 became NUL). The same code was correct where +/// wchar_t is 32 bits, so identical data behaved differently per platform. is also +/// deprecated since C++17 with no standard replacement, and this library's baseline is C++11. +/// +/// Conversion therefore pivots on Unicode code points here, and the wchar_t adapters in +/// StringUtilities select the UTF-16 or UTF-32 path by the width of wchar_t. +/// +/// Every function validates its input and throws std::range_error on anything malformed - +/// matching the behavior std::wstring_convert had for ill-formed sequences. Rejected are +/// overlong encodings, truncated sequences, stray or invalid lead and continuation bytes, +/// surrogate code points encoded directly (CESU-8), unpaired UTF-16 surrogates, and anything +/// above U+10FFFF. +class REFLECTION_EXPORT Utf8 { +public: + /// Encodes Unicode code points as UTF-8 + static std::string FromCodePoints(const char32_t* code_points, size_t count); + + /// Decodes UTF-8 into Unicode code points + static std::u32string ToCodePoints(const char* utf8_string, size_t byte_count); + + /// Encodes Unicode code points as UTF-16 code units, using a surrogate pair above U+FFFF + static std::u16string Utf16FromCodePoints(const char32_t* code_points, size_t count); + + /// Decodes UTF-16 code units, combining surrogate pairs, into Unicode code points + static std::u32string CodePointsFromUtf16(const char16_t* utf16_string, size_t count); +}; +} // namespace sqlite_reflection diff --git a/src/string_utilities.cc b/src/string_utilities.cc index c7f6d1c..8c03d29 100644 --- a/src/string_utilities.cc +++ b/src/string_utilities.cc @@ -22,14 +22,66 @@ #include "internal/string_utilities.h" -#include #include -#ifndef _WIN32 -#include -#endif + +#include "internal/utf8.h" using namespace sqlite_reflection; +namespace { +// wchar_t holds UTF-16 code units where it is 16 bits wide (Windows) and Unicode code points +// where it is 32 bits wide (Linux, macOS). Selecting the path by width is what keeps text above +// U+FFFF identical across platforms; a single facet cannot serve both, which is the defect this +// replaced. See internal/utf8.h. +template +struct WideCodec; + +template <> +struct WideCodec<2> { + static std::u32string ToCodePoints(const std::wstring& wide_string) { + std::u16string code_units; + code_units.reserve(wide_string.size()); + for (size_t i = 0; i < wide_string.size(); ++i) { + code_units.push_back(static_cast(wide_string[i])); + } + return Utf8::CodePointsFromUtf16(code_units.data(), code_units.size()); + } + + static std::wstring FromCodePoints(const std::u32string& code_points) { + const auto code_units = Utf8::Utf16FromCodePoints(code_points.data(), code_points.size()); + std::wstring wide_string; + wide_string.reserve(code_units.size()); + for (size_t i = 0; i < code_units.size(); ++i) { + wide_string.push_back(static_cast(code_units[i])); + } + return wide_string; + } +}; + +template <> +struct WideCodec<4> { + static std::u32string ToCodePoints(const std::wstring& wide_string) { + std::u32string code_points; + code_points.reserve(wide_string.size()); + for (size_t i = 0; i < wide_string.size(); ++i) { + code_points.push_back(static_cast(wide_string[i])); + } + return code_points; + } + + static std::wstring FromCodePoints(const std::u32string& code_points) { + std::wstring wide_string; + wide_string.reserve(code_points.size()); + for (size_t i = 0; i < code_points.size(); ++i) { + wide_string.push_back(static_cast(code_points[i])); + } + return wide_string; + } +}; + +typedef WideCodec PlatformWideCodec; +} // namespace + std::string StringUtilities::FromInt(int64_t value) { return std::to_string(value); } @@ -45,15 +97,13 @@ std::string StringUtilities::FromDouble(double value) { } std::string StringUtilities::ToUtf8(const std::wstring& wide_string) { - std::wstring_convert> converter; - auto utf8_string = converter.to_bytes(wide_string.data(), wide_string.data() + wide_string.size()); - return utf8_string; + const auto code_points = PlatformWideCodec::ToCodePoints(wide_string); + return Utf8::FromCodePoints(code_points.data(), code_points.size()); } std::wstring StringUtilities::FromUtf8(const char* utf8_string, size_t byte_count) { - std::wstring_convert> converter; - auto wide_string = converter.from_bytes(utf8_string, utf8_string + byte_count); - return wide_string; + const auto code_points = Utf8::ToCodePoints(utf8_string, byte_count); + return PlatformWideCodec::FromCodePoints(code_points); } std::string StringUtilities::Join(const std::vector& list, const std::string& separator) { diff --git a/src/utf8.cc b/src/utf8.cc new file mode 100644 index 0000000..5e1115c --- /dev/null +++ b/src/utf8.cc @@ -0,0 +1,235 @@ +// MIT License +// +// Copyright (c) 2026 Ioannis Kaliakatsos +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "internal/utf8.h" + +#include + +using namespace sqlite_reflection; + +namespace { +const char32_t kMaxCodePoint = 0x10FFFF; +const char32_t kFirstSurrogate = 0xD800; +const char32_t kLastSurrogate = 0xDFFF; +const char32_t kFirstSupplementary = 0x10000; + +const char16_t kFirstHighSurrogate = 0xD800; +const char16_t kLastHighSurrogate = 0xDBFF; +const char16_t kFirstLowSurrogate = 0xDC00; +const char16_t kLastLowSurrogate = 0xDFFF; + +void reject(const char* reason) { + throw std::range_error(std::string("invalid Unicode input: ") + reason); +} + +bool is_surrogate(char32_t code_point) { + return code_point >= kFirstSurrogate && code_point <= kLastSurrogate; +} + +// A code point that no valid encoding may carry: surrogates are reserved for UTF-16 pairing +// and must never appear on their own, and nothing above U+10FFFF exists. +void reject_if_not_scalar_value(char32_t code_point) { + if (is_surrogate(code_point)) { + reject("surrogate code point"); + } + if (code_point > kMaxCodePoint) { + reject("code point above U+10FFFF"); + } +} + +bool is_continuation(unsigned char byte) { + return (byte & 0xC0) == 0x80; +} + +// Length of the sequence introduced by this lead byte, or 0 if it cannot introduce one. +// 0x80-0xBF are continuation bytes, 0xC0-0xC1 could only encode overlong ASCII, and 0xF5-0xFF +// could only encode code points above U+10FFFF, so none of them are valid leads. +size_t sequence_length(unsigned char lead) { + if (lead <= 0x7F) { + return 1; + } + if (lead >= 0xC2 && lead <= 0xDF) { + return 2; + } + if (lead >= 0xE0 && lead <= 0xEF) { + return 3; + } + if (lead >= 0xF0 && lead <= 0xF4) { + return 4; + } + return 0; +} + +// Ranges of the second byte that would otherwise admit an overlong encoding, a surrogate, or a +// code point above U+10FFFF. The lead byte alone cannot exclude these (Unicode Table 3-7). +void reject_if_invalid_second_byte(unsigned char lead, unsigned char second) { + if (lead == 0xE0 && second < 0xA0) { + reject("overlong three-byte sequence"); + } + if (lead == 0xED && second > 0x9F) { + reject("surrogate code point encoded in UTF-8"); + } + if (lead == 0xF0 && second < 0x90) { + reject("overlong four-byte sequence"); + } + if (lead == 0xF4 && second > 0x8F) { + reject("code point above U+10FFFF"); + } +} +} // namespace + +std::string Utf8::FromCodePoints(const char32_t* code_points, size_t count) { + std::string utf8_string; + utf8_string.reserve(count); + + for (size_t i = 0; i < count; ++i) { + const char32_t code_point = code_points[i]; + reject_if_not_scalar_value(code_point); + + if (code_point <= 0x7F) { + utf8_string.push_back(static_cast(code_point)); + } else if (code_point <= 0x7FF) { + utf8_string.push_back(static_cast(0xC0 | (code_point >> 6))); + utf8_string.push_back(static_cast(0x80 | (code_point & 0x3F))); + } else if (code_point <= 0xFFFF) { + utf8_string.push_back(static_cast(0xE0 | (code_point >> 12))); + utf8_string.push_back(static_cast(0x80 | ((code_point >> 6) & 0x3F))); + utf8_string.push_back(static_cast(0x80 | (code_point & 0x3F))); + } else { + utf8_string.push_back(static_cast(0xF0 | (code_point >> 18))); + utf8_string.push_back(static_cast(0x80 | ((code_point >> 12) & 0x3F))); + utf8_string.push_back(static_cast(0x80 | ((code_point >> 6) & 0x3F))); + utf8_string.push_back(static_cast(0x80 | (code_point & 0x3F))); + } + } + + return utf8_string; +} + +std::u32string Utf8::ToCodePoints(const char* utf8_string, size_t byte_count) { + std::u32string code_points; + code_points.reserve(byte_count); + + size_t i = 0; + while (i < byte_count) { + const auto lead = static_cast(utf8_string[i]); + const size_t length = sequence_length(lead); + if (length == 0) { + reject("byte cannot start a UTF-8 sequence"); + } + if (i + length > byte_count) { + reject("truncated UTF-8 sequence"); + } + + if (length == 1) { + code_points.push_back(static_cast(lead)); + i += 1; + continue; + } + + const auto second = static_cast(utf8_string[i + 1]); + if (!is_continuation(second)) { + reject("missing UTF-8 continuation byte"); + } + reject_if_invalid_second_byte(lead, second); + + char32_t code_point = 0; + if (length == 2) { + code_point = static_cast(lead & 0x1F); + } else if (length == 3) { + code_point = static_cast(lead & 0x0F); + } else { + code_point = static_cast(lead & 0x07); + } + + for (size_t j = 1; j < length; ++j) { + const auto byte = static_cast(utf8_string[i + j]); + if (!is_continuation(byte)) { + reject("missing UTF-8 continuation byte"); + } + code_point = (code_point << 6) | static_cast(byte & 0x3F); + } + + reject_if_not_scalar_value(code_point); + code_points.push_back(code_point); + i += length; + } + + return code_points; +} + +std::u16string Utf8::Utf16FromCodePoints(const char32_t* code_points, size_t count) { + std::u16string utf16_string; + utf16_string.reserve(count); + + for (size_t i = 0; i < count; ++i) { + const char32_t code_point = code_points[i]; + reject_if_not_scalar_value(code_point); + + if (code_point < kFirstSupplementary) { + utf16_string.push_back(static_cast(code_point)); + continue; + } + + const char32_t offset = code_point - kFirstSupplementary; + utf16_string.push_back(static_cast(kFirstHighSurrogate + (offset >> 10))); + utf16_string.push_back(static_cast(kFirstLowSurrogate + (offset & 0x3FF))); + } + + return utf16_string; +} + +std::u32string Utf8::CodePointsFromUtf16(const char16_t* utf16_string, size_t count) { + std::u32string code_points; + code_points.reserve(count); + + size_t i = 0; + while (i < count) { + const char16_t unit = utf16_string[i]; + + if (unit >= kFirstLowSurrogate && unit <= kLastLowSurrogate) { + reject("unpaired low surrogate"); + } + + if (unit < kFirstHighSurrogate || unit > kLastHighSurrogate) { + code_points.push_back(static_cast(unit)); + i += 1; + continue; + } + + if (i + 1 >= count) { + reject("high surrogate at end of input"); + } + + const char16_t low = utf16_string[i + 1]; + if (low < kFirstLowSurrogate || low > kLastLowSurrogate) { + reject("high surrogate not followed by a low surrogate"); + } + + const char32_t high_bits = static_cast(unit - kFirstHighSurrogate) << 10; + const char32_t low_bits = static_cast(low - kFirstLowSurrogate); + code_points.push_back(kFirstSupplementary + high_bits + low_bits); + i += 2; + } + + return code_points; +} diff --git a/tests/utf8_test.cc b/tests/utf8_test.cc index a390953..d951b89 100644 --- a/tests/utf8_test.cc +++ b/tests/utf8_test.cc @@ -20,8 +20,11 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. +#include "internal/utf8.h" + #include +#include #include #include "internal/string_utilities.h" @@ -123,3 +126,119 @@ TEST(Utf8Test, SupplementaryCodePointSurvivesLengthAndContent) { EXPECT_FALSE(decoded.empty()); EXPECT_EQ(std::wstring(L"\U0001F600"), decoded); } + +// --------------------------------------------------------------------------- +// Code-point core, exercised directly so both the UTF-16 and UTF-32 paths are covered on every +// platform rather than only the one matching this machine's wchar_t width. The UTF-16 path is +// where the replaced facet was wrong, and on Linux and macOS it is unreachable +// through the wstring API. +// --------------------------------------------------------------------------- + +namespace { +std::u32string CodePoints(const std::string& utf8) { + return Utf8::ToCodePoints(utf8.data(), utf8.size()); +} + +std::string Utf8From(const std::u32string& code_points) { + return Utf8::FromCodePoints(code_points.data(), code_points.size()); +} + +std::u16string Utf16From(const std::u32string& code_points) { + return Utf8::Utf16FromCodePoints(code_points.data(), code_points.size()); +} + +std::u32string FromUtf16(const std::u16string& code_units) { + return Utf8::CodePointsFromUtf16(code_units.data(), code_units.size()); +} +} // namespace + +TEST(Utf8CoreTest, EncodesSupplementaryCodePointAsSurrogatePair) { + const std::u32string emoji(1, 0x1F600); + const std::u16string units = Utf16From(emoji); + ASSERT_EQ(static_cast(2), units.size()); + EXPECT_EQ(0xD83D, units[0]); + EXPECT_EQ(0xDE00, units[1]); +} + +TEST(Utf8CoreTest, CombinesSurrogatePairIntoSupplementaryCodePoint) { + std::u16string units; + units.push_back(0xD83D); + units.push_back(0xDE00); + const std::u32string code_points = FromUtf16(units); + ASSERT_EQ(static_cast(1), code_points.size()); + EXPECT_EQ(0x1F600u, static_cast(code_points[0])); +} + +TEST(Utf8CoreTest, RoundTripsSurrogateBoundaryThroughUtf16) { + std::u32string code_points; + code_points.push_back(0xFFFF); // last BMP + code_points.push_back(0x10000); // first supplementary + code_points.push_back(0x10FFFF); + EXPECT_EQ(code_points, FromUtf16(Utf16From(code_points))); +} + +TEST(Utf8CoreTest, EncodesSupplementaryCodePointAsFourUtf8Bytes) { + EXPECT_EQ(std::string(kEmojiUtf8), Utf8From(std::u32string(1, 0x1F600))); +} + +TEST(Utf8CoreTest, RejectsUnpairedHighSurrogateInUtf16) { + EXPECT_THROW(FromUtf16(std::u16string(1, 0xD83D)), std::range_error); +} + +TEST(Utf8CoreTest, RejectsUnpairedLowSurrogateInUtf16) { + EXPECT_THROW(FromUtf16(std::u16string(1, 0xDE00)), std::range_error); +} + +TEST(Utf8CoreTest, RejectsHighSurrogateFollowedByNonSurrogate) { + std::u16string units; + units.push_back(0xD83D); + units.push_back(0x0041); + EXPECT_THROW(FromUtf16(units), std::range_error); +} + +TEST(Utf8CoreTest, RejectsSurrogateCodePointOnEncode) { + EXPECT_THROW(Utf8From(std::u32string(1, 0xD800)), std::range_error); + EXPECT_THROW(Utf16From(std::u32string(1, 0xDFFF)), std::range_error); +} + +TEST(Utf8CoreTest, RejectsCodePointAboveUnicodeMaximumOnEncode) { + EXPECT_THROW(Utf8From(std::u32string(1, 0x110000)), std::range_error); +} + +TEST(Utf8CoreTest, RejectsInvalidLeadByte) { + EXPECT_THROW(CodePoints("\xFF"), std::range_error); // 0xFF is never a lead byte + EXPECT_THROW(CodePoints("\xC0\xAF"), std::range_error); // overlong ASCII + EXPECT_THROW(CodePoints("\x80"), std::range_error); // stray continuation byte + EXPECT_THROW(CodePoints("\xF5\x80\x80\x80"), std::range_error); // above U+10FFFF +} + +TEST(Utf8CoreTest, RejectsTruncatedSequence) { + EXPECT_THROW(CodePoints("\xF0\x9F\x98"), std::range_error); // emoji missing its last byte + EXPECT_THROW(CodePoints("\xCF"), std::range_error); // two-byte lead, nothing follows +} + +TEST(Utf8CoreTest, RejectsMissingContinuationByte) { + EXPECT_THROW(CodePoints("\xF0\x9F" + "A" + "\x80"), + std::range_error); + EXPECT_THROW(CodePoints("\xCF" + "A"), + std::range_error); +} + +TEST(Utf8CoreTest, RejectsOverlongEncoding) { + EXPECT_THROW(CodePoints("\xE0\x80\xAF"), std::range_error); // overlong three-byte + EXPECT_THROW(CodePoints("\xF0\x80\x80\xAF"), std::range_error); // overlong four-byte +} + +TEST(Utf8CoreTest, RejectsSurrogateEncodedInUtf8) { + // CESU-8: exactly what the replaced facet emitted for supplementary code points, so it must + // not be silently accepted on the way back in either. + EXPECT_THROW(CodePoints("\xED\xA0\xBD"), std::range_error); + EXPECT_THROW(CodePoints("\xED\xA0\xBD\xED\xB8\x80"), std::range_error); +} + +TEST(Utf8CoreTest, RejectsCodePointAboveUnicodeMaximumOnDecode) { + EXPECT_THROW(CodePoints("\xF4\x90\x80\x80"), std::range_error); // U+110000 +} From 2dd7c5b01c5515ea1e6d93ffdb5855186ffcb4e3 Mon Sep 17 00:00:00 2001 From: jkaliak Date: Sun, 26 Jul 2026 17:34:57 +0200 Subject: [PATCH 3/3] Rename Utf8 to Unicode and address review findings (#25) Review follow-up on the non-BMP conversion fix. Behavior is unchanged apart from one error message and the two documentation corrections. - Rename Utf8 to Unicode, since the class also carries the UTF-16 conversions, and give all four methods one naming shape: Utf8FromCodePoints, CodePointsFromUtf8, Utf16FromCodePoints, CodePointsFromUtf16. Files and test suites renamed to match. - Collapse the four duplicated element-wise cast loops in the WideCodec specializations into a single cast_each helper. - static_assert the supported wchar_t widths, so an unsupported width reports a portability limit rather than an incomplete type inside a typedef. - Rename reject to throw_invalid_unicode and mark it [[noreturn]], so call sites read as terminating without consulting the callee. - Derive the surrogate range from the UTF-16 bounds instead of repeating D800/DFFF as a second pair of constants at char32_t width. - Name CESU-8 as the likely cause in the decode error, and document that text written by a pre-fix build where wchar_t is 16 bits no longer reads back. That data was stored ill-formed; because encoding and decoding were wrong in the same way, it only surfaces now. - Correct the README throw guarantee to TEXT values: a BLOB in a MEMBER_TEXT column is skipped like NULL and never decoded. Also note the one platform difference that survives, that wstring counts code units, so L"\U0001F600".size() is 2 where wchar_t is 16 bits and 1 where it is 32. Verified on macOS Clang at C++11 and C++20: 124/124 tests pass in both, clang-format clean on the changed files. Co-Authored-By: Claude Opus 5 --- README.md | 18 +++++- src/internal/{utf8.h => unicode.h} | 11 +++- src/string_utilities.cc | 57 ++++++++++--------- src/{utf8.cc => unicode.cc} | 46 +++++++-------- tests/{utf8_test.cc => unicode_test.cc} | 74 ++++++++++++------------- 5 files changed, 113 insertions(+), 93 deletions(-) rename src/internal/{utf8.h => unicode.h} (84%) rename src/{utf8.cc => unicode.cc} (81%) rename tests/{utf8_test.cc => unicode_test.cc} (78%) diff --git a/README.md b/README.md index ec90b45..e845e70 100644 --- a/README.md +++ b/README.md @@ -144,8 +144,22 @@ extension blocks, and text round-trips unchanged on every supported platform reg whether `wchar_t` is 16 bits (Windows) or 32 bits (Linux, macOS). Text that is not valid Unicode is rejected rather than silently altered: converting a -`std::wstring` containing unpaired UTF-16 surrogates, or reading a column whose bytes are not -well-formed UTF-8, throws `std::range_error`. +`std::wstring` containing unpaired UTF-16 surrogates, or reading a TEXT value whose bytes are +not well-formed UTF-8, throws `std::range_error`. A column holding a BLOB where a +`MEMBER_TEXT` field is declared is skipped like a `NULL`, leaving the member untouched, so its +bytes are never decoded. + +Note that a `std::wstring` still counts code *units*, not code points, so `size()` and indexing +differ across platforms for text above U+FFFF: `L"\U0001F600".size()` is 2 where `wchar_t` is 16 +bits and 1 where it is 32 bits. What round-trips identically is the text itself. + +#### Reading databases written before this fix + +Earlier versions stored text above U+FFFF as CESU-8 when built where `wchar_t` is 16 bits +(Windows), encoding each surrogate half separately. Reading such a value now throws +`std::range_error` rather than returning it. Because the old encode and decode were wrong in the +same way, that data appeared to round-trip on Windows and was written ill-formed to the file; +databases written on Linux or macOS, and any text within the BMP, are unaffected. ### Nullable fields diff --git a/src/internal/utf8.h b/src/internal/unicode.h similarity index 84% rename from src/internal/utf8.h rename to src/internal/unicode.h index f3f14ad..d7c1414 100644 --- a/src/internal/utf8.h +++ b/src/internal/unicode.h @@ -46,13 +46,18 @@ namespace sqlite_reflection { /// overlong encodings, truncated sequences, stray or invalid lead and continuation bytes, /// surrogate code points encoded directly (CESU-8), unpaired UTF-16 surrogates, and anything /// above U+10FFFF. -class REFLECTION_EXPORT Utf8 { +/// +/// Note that rejecting CESU-8 means text written by a pre-fix build of this library, where +/// wchar_t is 16 bits, no longer reads back. That data was stored ill-formed; because encoding +/// and decoding were wrong in the same way it used to survive a round trip on Windows, so the +/// break surfaces only now. See the text encoding section of README.md. +class REFLECTION_EXPORT Unicode { public: /// Encodes Unicode code points as UTF-8 - static std::string FromCodePoints(const char32_t* code_points, size_t count); + static std::string Utf8FromCodePoints(const char32_t* code_points, size_t count); /// Decodes UTF-8 into Unicode code points - static std::u32string ToCodePoints(const char* utf8_string, size_t byte_count); + static std::u32string CodePointsFromUtf8(const char* utf8_string, size_t byte_count); /// Encodes Unicode code points as UTF-16 code units, using a surrogate pair above U+FFFF static std::u16string Utf16FromCodePoints(const char32_t* code_points, size_t count); diff --git a/src/string_utilities.cc b/src/string_utilities.cc index 8c03d29..948449e 100644 --- a/src/string_utilities.cc +++ b/src/string_utilities.cc @@ -24,61 +24,60 @@ #include -#include "internal/utf8.h" +#include "internal/unicode.h" using namespace sqlite_reflection; namespace { +// Copies a string of code units into a string of a different unit type, one unit at a time. The +// casts are value-preserving in the directions used below, except for a negative wchar_t on a +// platform where it is signed, which widens to a value above U+10FFFF and is then rejected by +// the validation in Unicode. +template +To cast_each(const From& source) { + To result; + result.reserve(source.size()); + for (size_t i = 0; i < source.size(); ++i) { + result.push_back(static_cast(source[i])); + } + return result; +} + // wchar_t holds UTF-16 code units where it is 16 bits wide (Windows) and Unicode code points // where it is 32 bits wide (Linux, macOS). Selecting the path by width is what keeps text above // U+FFFF identical across platforms; a single facet cannot serve both, which is the defect this -// replaced. See internal/utf8.h. +// replaced. See internal/unicode.h. template struct WideCodec; template <> struct WideCodec<2> { static std::u32string ToCodePoints(const std::wstring& wide_string) { - std::u16string code_units; - code_units.reserve(wide_string.size()); - for (size_t i = 0; i < wide_string.size(); ++i) { - code_units.push_back(static_cast(wide_string[i])); - } - return Utf8::CodePointsFromUtf16(code_units.data(), code_units.size()); + const auto code_units = cast_each(wide_string); + return Unicode::CodePointsFromUtf16(code_units.data(), code_units.size()); } static std::wstring FromCodePoints(const std::u32string& code_points) { - const auto code_units = Utf8::Utf16FromCodePoints(code_points.data(), code_points.size()); - std::wstring wide_string; - wide_string.reserve(code_units.size()); - for (size_t i = 0; i < code_units.size(); ++i) { - wide_string.push_back(static_cast(code_units[i])); - } - return wide_string; + const auto code_units = Unicode::Utf16FromCodePoints(code_points.data(), code_points.size()); + return cast_each(code_units); } }; template <> struct WideCodec<4> { static std::u32string ToCodePoints(const std::wstring& wide_string) { - std::u32string code_points; - code_points.reserve(wide_string.size()); - for (size_t i = 0; i < wide_string.size(); ++i) { - code_points.push_back(static_cast(wide_string[i])); - } - return code_points; + return cast_each(wide_string); } static std::wstring FromCodePoints(const std::u32string& code_points) { - std::wstring wide_string; - wide_string.reserve(code_points.size()); - for (size_t i = 0; i < code_points.size(); ++i) { - wide_string.push_back(static_cast(code_points[i])); - } - return wide_string; + return cast_each(code_points); } }; +// Only the two widths above are implemented; without this the failure is an incomplete-type +// error inside the typedef rather than a statement about portability. +static_assert(sizeof(wchar_t) == 2 || sizeof(wchar_t) == 4, "sqlite_reflection supports wchar_t of 16 or 32 bits only"); + typedef WideCodec PlatformWideCodec; } // namespace @@ -98,11 +97,11 @@ std::string StringUtilities::FromDouble(double value) { std::string StringUtilities::ToUtf8(const std::wstring& wide_string) { const auto code_points = PlatformWideCodec::ToCodePoints(wide_string); - return Utf8::FromCodePoints(code_points.data(), code_points.size()); + return Unicode::Utf8FromCodePoints(code_points.data(), code_points.size()); } std::wstring StringUtilities::FromUtf8(const char* utf8_string, size_t byte_count) { - const auto code_points = Utf8::ToCodePoints(utf8_string, byte_count); + const auto code_points = Unicode::CodePointsFromUtf8(utf8_string, byte_count); return PlatformWideCodec::FromCodePoints(code_points); } diff --git a/src/utf8.cc b/src/unicode.cc similarity index 81% rename from src/utf8.cc rename to src/unicode.cc index 5e1115c..d80e57b 100644 --- a/src/utf8.cc +++ b/src/unicode.cc @@ -20,7 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "internal/utf8.h" +#include "internal/unicode.h" #include @@ -28,8 +28,6 @@ using namespace sqlite_reflection; namespace { const char32_t kMaxCodePoint = 0x10FFFF; -const char32_t kFirstSurrogate = 0xD800; -const char32_t kLastSurrogate = 0xDFFF; const char32_t kFirstSupplementary = 0x10000; const char16_t kFirstHighSurrogate = 0xD800; @@ -37,22 +35,24 @@ const char16_t kLastHighSurrogate = 0xDBFF; const char16_t kFirstLowSurrogate = 0xDC00; const char16_t kLastLowSurrogate = 0xDFFF; -void reject(const char* reason) { +[[noreturn]] void throw_invalid_unicode(const char* reason) { throw std::range_error(std::string("invalid Unicode input: ") + reason); } +// The surrogate range is exactly the first high through the last low surrogate; the char16_t +// operands widen to char32_t for the comparison. bool is_surrogate(char32_t code_point) { - return code_point >= kFirstSurrogate && code_point <= kLastSurrogate; + return code_point >= kFirstHighSurrogate && code_point <= kLastLowSurrogate; } // A code point that no valid encoding may carry: surrogates are reserved for UTF-16 pairing // and must never appear on their own, and nothing above U+10FFFF exists. void reject_if_not_scalar_value(char32_t code_point) { if (is_surrogate(code_point)) { - reject("surrogate code point"); + throw_invalid_unicode("surrogate code point"); } if (code_point > kMaxCodePoint) { - reject("code point above U+10FFFF"); + throw_invalid_unicode("code point above U+10FFFF"); } } @@ -83,21 +83,23 @@ size_t sequence_length(unsigned char lead) { // code point above U+10FFFF. The lead byte alone cannot exclude these (Unicode Table 3-7). void reject_if_invalid_second_byte(unsigned char lead, unsigned char second) { if (lead == 0xE0 && second < 0xA0) { - reject("overlong three-byte sequence"); + throw_invalid_unicode("overlong three-byte sequence"); } if (lead == 0xED && second > 0x9F) { - reject("surrogate code point encoded in UTF-8"); + throw_invalid_unicode( + "surrogate code point encoded in UTF-8 (CESU-8), as written by a pre-fix build of this " + "library where wchar_t is 16 bits"); } if (lead == 0xF0 && second < 0x90) { - reject("overlong four-byte sequence"); + throw_invalid_unicode("overlong four-byte sequence"); } if (lead == 0xF4 && second > 0x8F) { - reject("code point above U+10FFFF"); + throw_invalid_unicode("code point above U+10FFFF"); } } } // namespace -std::string Utf8::FromCodePoints(const char32_t* code_points, size_t count) { +std::string Unicode::Utf8FromCodePoints(const char32_t* code_points, size_t count) { std::string utf8_string; utf8_string.reserve(count); @@ -125,7 +127,7 @@ std::string Utf8::FromCodePoints(const char32_t* code_points, size_t count) { return utf8_string; } -std::u32string Utf8::ToCodePoints(const char* utf8_string, size_t byte_count) { +std::u32string Unicode::CodePointsFromUtf8(const char* utf8_string, size_t byte_count) { std::u32string code_points; code_points.reserve(byte_count); @@ -134,10 +136,10 @@ std::u32string Utf8::ToCodePoints(const char* utf8_string, size_t byte_count) { const auto lead = static_cast(utf8_string[i]); const size_t length = sequence_length(lead); if (length == 0) { - reject("byte cannot start a UTF-8 sequence"); + throw_invalid_unicode("byte cannot start a UTF-8 sequence"); } if (i + length > byte_count) { - reject("truncated UTF-8 sequence"); + throw_invalid_unicode("truncated UTF-8 sequence"); } if (length == 1) { @@ -148,7 +150,7 @@ std::u32string Utf8::ToCodePoints(const char* utf8_string, size_t byte_count) { const auto second = static_cast(utf8_string[i + 1]); if (!is_continuation(second)) { - reject("missing UTF-8 continuation byte"); + throw_invalid_unicode("missing UTF-8 continuation byte"); } reject_if_invalid_second_byte(lead, second); @@ -164,7 +166,7 @@ std::u32string Utf8::ToCodePoints(const char* utf8_string, size_t byte_count) { for (size_t j = 1; j < length; ++j) { const auto byte = static_cast(utf8_string[i + j]); if (!is_continuation(byte)) { - reject("missing UTF-8 continuation byte"); + throw_invalid_unicode("missing UTF-8 continuation byte"); } code_point = (code_point << 6) | static_cast(byte & 0x3F); } @@ -177,7 +179,7 @@ std::u32string Utf8::ToCodePoints(const char* utf8_string, size_t byte_count) { return code_points; } -std::u16string Utf8::Utf16FromCodePoints(const char32_t* code_points, size_t count) { +std::u16string Unicode::Utf16FromCodePoints(const char32_t* code_points, size_t count) { std::u16string utf16_string; utf16_string.reserve(count); @@ -198,7 +200,7 @@ std::u16string Utf8::Utf16FromCodePoints(const char32_t* code_points, size_t cou return utf16_string; } -std::u32string Utf8::CodePointsFromUtf16(const char16_t* utf16_string, size_t count) { +std::u32string Unicode::CodePointsFromUtf16(const char16_t* utf16_string, size_t count) { std::u32string code_points; code_points.reserve(count); @@ -207,7 +209,7 @@ std::u32string Utf8::CodePointsFromUtf16(const char16_t* utf16_string, size_t co const char16_t unit = utf16_string[i]; if (unit >= kFirstLowSurrogate && unit <= kLastLowSurrogate) { - reject("unpaired low surrogate"); + throw_invalid_unicode("unpaired low surrogate"); } if (unit < kFirstHighSurrogate || unit > kLastHighSurrogate) { @@ -217,12 +219,12 @@ std::u32string Utf8::CodePointsFromUtf16(const char16_t* utf16_string, size_t co } if (i + 1 >= count) { - reject("high surrogate at end of input"); + throw_invalid_unicode("high surrogate at end of input"); } const char16_t low = utf16_string[i + 1]; if (low < kFirstLowSurrogate || low > kLastLowSurrogate) { - reject("high surrogate not followed by a low surrogate"); + throw_invalid_unicode("high surrogate not followed by a low surrogate"); } const char32_t high_bits = static_cast(unit - kFirstHighSurrogate) << 10; diff --git a/tests/utf8_test.cc b/tests/unicode_test.cc similarity index 78% rename from tests/utf8_test.cc rename to tests/unicode_test.cc index d951b89..103154d 100644 --- a/tests/utf8_test.cc +++ b/tests/unicode_test.cc @@ -20,7 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "internal/utf8.h" +#include "internal/unicode.h" #include @@ -51,75 +51,75 @@ std::wstring FromUtf8(const std::string& utf8) { } } // namespace -TEST(Utf8Test, EncodesAsciiToExactBytes) { +TEST(UnicodeTest, EncodesAsciiToExactBytes) { EXPECT_EQ(std::string("Appleseed"), StringUtilities::ToUtf8(L"Appleseed")); } -TEST(Utf8Test, EncodesEmptyString) { +TEST(UnicodeTest, EncodesEmptyString) { EXPECT_EQ(std::string(), StringUtilities::ToUtf8(std::wstring())); } -TEST(Utf8Test, EncodesBmpToExactBytes) { +TEST(UnicodeTest, EncodesBmpToExactBytes) { // U+03C0 GREEK SMALL LETTER PI, U+03B1 GREEK SMALL LETTER ALPHA: two-byte sequences. EXPECT_EQ(std::string("\xCF\x80\xCE\xB1"), StringUtilities::ToUtf8(L"\u03C0\u03B1")); } -TEST(Utf8Test, EncodesLastBmpCodePointToExactBytes) { +TEST(UnicodeTest, EncodesLastBmpCodePointToExactBytes) { EXPECT_EQ(std::string(kLastBmpUtf8), StringUtilities::ToUtf8(L"\uFFFF")); } -TEST(Utf8Test, EncodesFirstSupplementaryCodePointToExactBytes) { +TEST(UnicodeTest, EncodesFirstSupplementaryCodePointToExactBytes) { // U+10000 is the low side of the surrogate boundary: the first code point that needs a // surrogate pair in UTF-16 and a four-byte UTF-8 sequence. EXPECT_EQ(std::string(kFirstSupplementaryUtf8), StringUtilities::ToUtf8(L"\U00010000")); } -TEST(Utf8Test, EncodesEmojiToExactBytes) { +TEST(UnicodeTest, EncodesEmojiToExactBytes) { EXPECT_EQ(std::string(kEmojiUtf8), StringUtilities::ToUtf8(L"\U0001F600")); } -TEST(Utf8Test, EncodesLastValidCodePointToExactBytes) { +TEST(UnicodeTest, EncodesLastValidCodePointToExactBytes) { EXPECT_EQ(std::string(kLastCodePointUtf8), StringUtilities::ToUtf8(L"\U0010FFFF")); } -TEST(Utf8Test, EncodesSupplementaryCodePointMixedWithAscii) { +TEST(UnicodeTest, EncodesSupplementaryCodePointMixedWithAscii) { EXPECT_EQ(std::string("a") + kEmojiUtf8 + "b", StringUtilities::ToUtf8(L"a\U0001F600b")); } -TEST(Utf8Test, DecodesAscii) { +TEST(UnicodeTest, DecodesAscii) { EXPECT_EQ(std::wstring(L"Appleseed"), FromUtf8("Appleseed")); } -TEST(Utf8Test, DecodesEmptyString) { +TEST(UnicodeTest, DecodesEmptyString) { EXPECT_EQ(std::wstring(), FromUtf8(std::string())); } -TEST(Utf8Test, DecodesBmp) { +TEST(UnicodeTest, DecodesBmp) { EXPECT_EQ(std::wstring(L"\u03C0\u03B1"), FromUtf8("\xCF\x80\xCE\xB1")); } -TEST(Utf8Test, DecodesLastBmpCodePoint) { +TEST(UnicodeTest, DecodesLastBmpCodePoint) { EXPECT_EQ(std::wstring(L"\uFFFF"), FromUtf8(kLastBmpUtf8)); } -TEST(Utf8Test, DecodesFirstSupplementaryCodePoint) { +TEST(UnicodeTest, DecodesFirstSupplementaryCodePoint) { EXPECT_EQ(std::wstring(L"\U00010000"), FromUtf8(kFirstSupplementaryUtf8)); } -TEST(Utf8Test, DecodesEmoji) { +TEST(UnicodeTest, DecodesEmoji) { EXPECT_EQ(std::wstring(L"\U0001F600"), FromUtf8(kEmojiUtf8)); } -TEST(Utf8Test, DecodesLastValidCodePoint) { +TEST(UnicodeTest, DecodesLastValidCodePoint) { EXPECT_EQ(std::wstring(L"\U0010FFFF"), FromUtf8(kLastCodePointUtf8)); } -TEST(Utf8Test, RoundTripsSupplementaryCodePoints) { +TEST(UnicodeTest, RoundTripsSupplementaryCodePoints) { const std::wstring original = L"\U0001F600\U00010000\U0010FFFF"; EXPECT_EQ(original, FromUtf8(StringUtilities::ToUtf8(original))); } -TEST(Utf8Test, SupplementaryCodePointSurvivesLengthAndContent) { +TEST(UnicodeTest, SupplementaryCodePointSurvivesLengthAndContent) { // Guards against a conversion that silently drops or truncates the pair rather than // producing a wrong-but-present result. const std::wstring decoded = FromUtf8(kEmojiUtf8); @@ -136,23 +136,23 @@ TEST(Utf8Test, SupplementaryCodePointSurvivesLengthAndContent) { namespace { std::u32string CodePoints(const std::string& utf8) { - return Utf8::ToCodePoints(utf8.data(), utf8.size()); + return Unicode::CodePointsFromUtf8(utf8.data(), utf8.size()); } std::string Utf8From(const std::u32string& code_points) { - return Utf8::FromCodePoints(code_points.data(), code_points.size()); + return Unicode::Utf8FromCodePoints(code_points.data(), code_points.size()); } std::u16string Utf16From(const std::u32string& code_points) { - return Utf8::Utf16FromCodePoints(code_points.data(), code_points.size()); + return Unicode::Utf16FromCodePoints(code_points.data(), code_points.size()); } std::u32string FromUtf16(const std::u16string& code_units) { - return Utf8::CodePointsFromUtf16(code_units.data(), code_units.size()); + return Unicode::CodePointsFromUtf16(code_units.data(), code_units.size()); } } // namespace -TEST(Utf8CoreTest, EncodesSupplementaryCodePointAsSurrogatePair) { +TEST(UnicodeCoreTest, EncodesSupplementaryCodePointAsSurrogatePair) { const std::u32string emoji(1, 0x1F600); const std::u16string units = Utf16From(emoji); ASSERT_EQ(static_cast(2), units.size()); @@ -160,7 +160,7 @@ TEST(Utf8CoreTest, EncodesSupplementaryCodePointAsSurrogatePair) { EXPECT_EQ(0xDE00, units[1]); } -TEST(Utf8CoreTest, CombinesSurrogatePairIntoSupplementaryCodePoint) { +TEST(UnicodeCoreTest, CombinesSurrogatePairIntoSupplementaryCodePoint) { std::u16string units; units.push_back(0xD83D); units.push_back(0xDE00); @@ -169,7 +169,7 @@ TEST(Utf8CoreTest, CombinesSurrogatePairIntoSupplementaryCodePoint) { EXPECT_EQ(0x1F600u, static_cast(code_points[0])); } -TEST(Utf8CoreTest, RoundTripsSurrogateBoundaryThroughUtf16) { +TEST(UnicodeCoreTest, RoundTripsSurrogateBoundaryThroughUtf16) { std::u32string code_points; code_points.push_back(0xFFFF); // last BMP code_points.push_back(0x10000); // first supplementary @@ -177,47 +177,47 @@ TEST(Utf8CoreTest, RoundTripsSurrogateBoundaryThroughUtf16) { EXPECT_EQ(code_points, FromUtf16(Utf16From(code_points))); } -TEST(Utf8CoreTest, EncodesSupplementaryCodePointAsFourUtf8Bytes) { +TEST(UnicodeCoreTest, EncodesSupplementaryCodePointAsFourUtf8Bytes) { EXPECT_EQ(std::string(kEmojiUtf8), Utf8From(std::u32string(1, 0x1F600))); } -TEST(Utf8CoreTest, RejectsUnpairedHighSurrogateInUtf16) { +TEST(UnicodeCoreTest, RejectsUnpairedHighSurrogateInUtf16) { EXPECT_THROW(FromUtf16(std::u16string(1, 0xD83D)), std::range_error); } -TEST(Utf8CoreTest, RejectsUnpairedLowSurrogateInUtf16) { +TEST(UnicodeCoreTest, RejectsUnpairedLowSurrogateInUtf16) { EXPECT_THROW(FromUtf16(std::u16string(1, 0xDE00)), std::range_error); } -TEST(Utf8CoreTest, RejectsHighSurrogateFollowedByNonSurrogate) { +TEST(UnicodeCoreTest, RejectsHighSurrogateFollowedByNonSurrogate) { std::u16string units; units.push_back(0xD83D); units.push_back(0x0041); EXPECT_THROW(FromUtf16(units), std::range_error); } -TEST(Utf8CoreTest, RejectsSurrogateCodePointOnEncode) { +TEST(UnicodeCoreTest, RejectsSurrogateCodePointOnEncode) { EXPECT_THROW(Utf8From(std::u32string(1, 0xD800)), std::range_error); EXPECT_THROW(Utf16From(std::u32string(1, 0xDFFF)), std::range_error); } -TEST(Utf8CoreTest, RejectsCodePointAboveUnicodeMaximumOnEncode) { +TEST(UnicodeCoreTest, RejectsCodePointAboveUnicodeMaximumOnEncode) { EXPECT_THROW(Utf8From(std::u32string(1, 0x110000)), std::range_error); } -TEST(Utf8CoreTest, RejectsInvalidLeadByte) { +TEST(UnicodeCoreTest, RejectsInvalidLeadByte) { EXPECT_THROW(CodePoints("\xFF"), std::range_error); // 0xFF is never a lead byte EXPECT_THROW(CodePoints("\xC0\xAF"), std::range_error); // overlong ASCII EXPECT_THROW(CodePoints("\x80"), std::range_error); // stray continuation byte EXPECT_THROW(CodePoints("\xF5\x80\x80\x80"), std::range_error); // above U+10FFFF } -TEST(Utf8CoreTest, RejectsTruncatedSequence) { +TEST(UnicodeCoreTest, RejectsTruncatedSequence) { EXPECT_THROW(CodePoints("\xF0\x9F\x98"), std::range_error); // emoji missing its last byte EXPECT_THROW(CodePoints("\xCF"), std::range_error); // two-byte lead, nothing follows } -TEST(Utf8CoreTest, RejectsMissingContinuationByte) { +TEST(UnicodeCoreTest, RejectsMissingContinuationByte) { EXPECT_THROW(CodePoints("\xF0\x9F" "A" "\x80"), @@ -227,18 +227,18 @@ TEST(Utf8CoreTest, RejectsMissingContinuationByte) { std::range_error); } -TEST(Utf8CoreTest, RejectsOverlongEncoding) { +TEST(UnicodeCoreTest, RejectsOverlongEncoding) { EXPECT_THROW(CodePoints("\xE0\x80\xAF"), std::range_error); // overlong three-byte EXPECT_THROW(CodePoints("\xF0\x80\x80\xAF"), std::range_error); // overlong four-byte } -TEST(Utf8CoreTest, RejectsSurrogateEncodedInUtf8) { +TEST(UnicodeCoreTest, RejectsSurrogateEncodedInUtf8) { // CESU-8: exactly what the replaced facet emitted for supplementary code points, so it must // not be silently accepted on the way back in either. EXPECT_THROW(CodePoints("\xED\xA0\xBD"), std::range_error); EXPECT_THROW(CodePoints("\xED\xA0\xBD\xED\xB8\x80"), std::range_error); } -TEST(Utf8CoreTest, RejectsCodePointAboveUnicodeMaximumOnDecode) { +TEST(UnicodeCoreTest, RejectsCodePointAboveUnicodeMaximumOnDecode) { EXPECT_THROW(CodePoints("\xF4\x90\x80\x80"), std::range_error); // U+110000 }