diff --git a/README.md b/README.md index 44dd969..5865f98 100644 --- a/README.md +++ b/README.md @@ -669,6 +669,8 @@ to the expected behavior. `last_write_time()` now consistently follows symlinks. * Fix for [#208](https://github.com/gulrak/filesystem/issues/208), filesystem metadata now preserves subsecond modification times. +* Fix for [#209](https://github.com/gulrak/filesystem/issues/209), malformed + UTF-16 no longer drops subsequent valid code units. * Fix for [#178](https://github.com/gulrak/filesystem/issues/178), recursive iteration no longer resolves symlinks unless requested. * Fix for [#185](https://github.com/gulrak/filesystem/issues/185), diff --git a/include/ghc/filesystem.hpp b/include/ghc/filesystem.hpp index b47e40d..3015304 100644 --- a/include/ghc/filesystem.hpp +++ b/include/ghc/filesystem.hpp @@ -1656,22 +1656,28 @@ inline std::string toUtf8(const strT& unicodeString) std::string result; for (auto iter = unicodeString.begin(); iter != unicodeString.end(); ++iter) { char32_t c = *iter; - if (is_surrogate(c)) { - ++iter; - if (iter != unicodeString.end() && is_high_surrogate(c) && is_low_surrogate(*iter)) { - appendUTF8(result, (char32_t(c) << 10) + *iter - 0x35fdc00); + if (is_high_surrogate(c)) { + auto next = iter; + ++next; + if (next != unicodeString.end() && is_low_surrogate(*next)) { + appendUTF8(result, (char32_t(c) << 10) + *next - 0x35fdc00); + iter = next; } else { #ifdef GHC_RAISE_UNICODE_ERRORS throw filesystem_error("Illegal code point for unicode character.", result, std::make_error_code(std::errc::illegal_byte_sequence)); #else appendUTF8(result, 0xfffd); - if (iter == unicodeString.end()) { - break; - } #endif } } + else if (is_low_surrogate(c)) { +#ifdef GHC_RAISE_UNICODE_ERRORS + throw filesystem_error("Illegal code point for unicode character.", result, std::make_error_code(std::errc::illegal_byte_sequence)); +#else + appendUTF8(result, 0xfffd); +#endif + } else { appendUTF8(result, c); } diff --git a/test/filesystem_test.cpp b/test/filesystem_test.cpp index f72ec94..127f52e 100644 --- a/test/filesystem_test.cpp +++ b/test/filesystem_test.cpp @@ -355,11 +355,23 @@ TEST_CASE("fs::detail::toUtf8", "[filesystem][fs.detail.utf8]") { std::string t; CHECK(std::string("\xc3\xa4/\xe2\x82\xac\xf0\x9d\x84\x9e") == fs::detail::toUtf8(std::u16string(u"\u00E4/\u20AC\U0001D11E"))); + const std::u16string highThenAscii{0xd800, u'A'}; + const std::u16string lowThenAscii{0xdc00, u'B'}; + const std::u16string malformedThenPair{0xd800, 0xd834, 0xdd1e, u'C'}; + const std::u16string pairThenMalformed{0xd834, 0xdd1e, 0xdc00, u'D'}; #ifdef GHC_RAISE_UNICODE_ERRORS CHECK_THROWS_AS(fs::detail::toUtf8(std::u16string(1, 0xd800)), fs::filesystem_error); + CHECK_THROWS_AS(fs::detail::toUtf8(highThenAscii), fs::filesystem_error); + CHECK_THROWS_AS(fs::detail::toUtf8(lowThenAscii), fs::filesystem_error); + CHECK_THROWS_AS(fs::detail::toUtf8(malformedThenPair), fs::filesystem_error); + CHECK_THROWS_AS(fs::detail::toUtf8(pairThenMalformed), fs::filesystem_error); CHECK_THROWS_AS(fs::detail::appendUTF8(t, 0x200000), fs::filesystem_error); #else CHECK(std::string("\xEF\xBF\xBD") == fs::detail::toUtf8(std::u16string(1, 0xd800))); + CHECK(std::string("\xEF\xBF\xBD" "A") == fs::detail::toUtf8(highThenAscii)); + CHECK(std::string("\xEF\xBF\xBD" "B") == fs::detail::toUtf8(lowThenAscii)); + CHECK(std::string("\xEF\xBF\xBD\xF0\x9D\x84\x9E" "C") == fs::detail::toUtf8(malformedThenPair)); + CHECK(std::string("\xF0\x9D\x84\x9E\xEF\xBF\xBD" "D") == fs::detail::toUtf8(pairThenMalformed)); fs::detail::appendUTF8(t, 0x200000); CHECK(std::string("\xEF\xBF\xBD") == t); #endif