From c5ebc28232b3eb987372f779439a03f45e26e25c Mon Sep 17 00:00:00 2001 From: Patrick Schmidt Date: Wed, 16 Jan 2019 21:16:51 +0100 Subject: [PATCH] #1020 handle null bytes when parsing utf8 (#1021) * add regression test for conversion of null byte * handle null bytes like single byte characters --- Release/src/utilities/asyncrt_utils.cpp | 4 ++-- Release/tests/functional/utils/strings.cpp | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Release/src/utilities/asyncrt_utils.cpp b/Release/src/utilities/asyncrt_utils.cpp index 426977478..96f3d9af9 100644 --- a/Release/src/utilities/asyncrt_utils.cpp +++ b/Release/src/utilities/asyncrt_utils.cpp @@ -341,11 +341,11 @@ inline size_t count_utf8_to_utf16(const std::string& s) for (size_t index = 0; index < sSize;) { - if (sData[index] > 0) + if (sData[index] >= 0) { // use fast inner loop to skip single byte code points (which are // expected to be the most frequent) - while ((++index < sSize) && (sData[index] > 0)) + while ((++index < sSize) && (sData[index] >= 0)) ; if (index >= sSize) break; diff --git a/Release/tests/functional/utils/strings.cpp b/Release/tests/functional/utils/strings.cpp index 1879e1966..261714333 100644 --- a/Release/tests/functional/utils/strings.cpp +++ b/Release/tests/functional/utils/strings.cpp @@ -153,6 +153,14 @@ SUITE(strings) auto result = utility::conversions::utf8_to_utf16(input); VERIFY_ARE_EQUAL(0x7F, result[0]); + // null byte + input.clear(); + input.push_back(0); + input.push_back(0); + result = utility::conversions::utf8_to_utf16(input); + VERIFY_ARE_EQUAL(0, result[0]); + VERIFY_ARE_EQUAL(0, result[1]); + // 2 byte character input.clear(); // U+80