Handle malformed UTF-16 sequences in toUtf8 conversion and update error handling, refs #209.

README.md

@@ -669,6 +669,8 @@ to the expected behavior.
   `last_write_time()` now consistently follows symlinks.
 * Fix for [#208](https://github.com/gulrak/filesystem/issues/208), filesystem
   metadata now preserves subsecond modification times.
+* Fix for [#209](https://github.com/gulrak/filesystem/issues/209), malformed
+  UTF-16 no longer drops subsequent valid code units.
 * Fix for [#178](https://github.com/gulrak/filesystem/issues/178), recursive
   iteration no longer resolves symlinks unless requested.
 * Fix for [#185](https://github.com/gulrak/filesystem/issues/185),

include/ghc/filesystem.hpp

@@ -1656,22 +1656,28 @@ inline std::string toUtf8(const strT& unicodeString)
     std::string result;
     for (auto iter = unicodeString.begin(); iter != unicodeString.end(); ++iter) {
         char32_t c = *iter;
-        if (is_surrogate(c)) {
+        if (is_high_surrogate(c)) {
-            ++iter;
+            auto next = iter;
-            if (iter != unicodeString.end() && is_high_surrogate(c) && is_low_surrogate(*iter)) {
+            ++next;
-                appendUTF8(result, (char32_t(c) << 10) + *iter - 0x35fdc00);
+            if (next != unicodeString.end() && is_low_surrogate(*next)) {
+                appendUTF8(result, (char32_t(c) << 10) + *next - 0x35fdc00);
+                iter = next;
             }
             else {
 #ifdef GHC_RAISE_UNICODE_ERRORS
                 throw filesystem_error("Illegal code point for unicode character.", result, std::make_error_code(std::errc::illegal_byte_sequence));
 #else
                 appendUTF8(result, 0xfffd);
-                if (iter == unicodeString.end()) {
-                    break;
-                }
 #endif
             }
         }
+        else if (is_low_surrogate(c)) {
+#ifdef GHC_RAISE_UNICODE_ERRORS
+            throw filesystem_error("Illegal code point for unicode character.", result, std::make_error_code(std::errc::illegal_byte_sequence));
+#else
+            appendUTF8(result, 0xfffd);
+#endif
+        }
         else {
             appendUTF8(result, c);
         }

test/filesystem_test.cpp

@@ -355,11 +355,23 @@ TEST_CASE("fs::detail::toUtf8", "[filesystem][fs.detail.utf8]")
 {
     std::string t;
     CHECK(std::string("\xc3\xa4/\xe2\x82\xac\xf0\x9d\x84\x9e") == fs::detail::toUtf8(std::u16string(u"\u00E4/\u20AC\U0001D11E")));
+    const std::u16string highThenAscii{0xd800, u'A'};
+    const std::u16string lowThenAscii{0xdc00, u'B'};
+    const std::u16string malformedThenPair{0xd800, 0xd834, 0xdd1e, u'C'};
+    const std::u16string pairThenMalformed{0xd834, 0xdd1e, 0xdc00, u'D'};
 #ifdef GHC_RAISE_UNICODE_ERRORS
     CHECK_THROWS_AS(fs::detail::toUtf8(std::u16string(1, 0xd800)), fs::filesystem_error);
+    CHECK_THROWS_AS(fs::detail::toUtf8(highThenAscii), fs::filesystem_error);
+    CHECK_THROWS_AS(fs::detail::toUtf8(lowThenAscii), fs::filesystem_error);
+    CHECK_THROWS_AS(fs::detail::toUtf8(malformedThenPair), fs::filesystem_error);
+    CHECK_THROWS_AS(fs::detail::toUtf8(pairThenMalformed), fs::filesystem_error);
     CHECK_THROWS_AS(fs::detail::appendUTF8(t, 0x200000), fs::filesystem_error);
 #else
     CHECK(std::string("\xEF\xBF\xBD") == fs::detail::toUtf8(std::u16string(1, 0xd800)));
+    CHECK(std::string("\xEF\xBF\xBD" "A") == fs::detail::toUtf8(highThenAscii));
+    CHECK(std::string("\xEF\xBF\xBD" "B") == fs::detail::toUtf8(lowThenAscii));
+    CHECK(std::string("\xEF\xBF\xBD\xF0\x9D\x84\x9E" "C") == fs::detail::toUtf8(malformedThenPair));
+    CHECK(std::string("\xF0\x9D\x84\x9E\xEF\xBF\xBD" "D") == fs::detail::toUtf8(pairThenMalformed));
     fs::detail::appendUTF8(t, 0x200000);
     CHECK(std::string("\xEF\xBF\xBD") == t);
 #endif