coffee/fast_float
1
0
Fork 0
mirror of https://github.com/fastfloat/fast_float.git synced 2025-12-06 08:46:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Do not mis-parse certain wide-character emojis as integer

Browse Source 
When calling ch_to_digit() with a UTF-16 or UTF-32 code unit, it simply
truncates away any data stored in the non-low byte(s) of the code unit.
It then uses a lookup table to determine whether the low byte
corresponds to an ASCII digit. This is incorrect because as soon as any
bit outside the low byte is set, the number will never correspond to a
ASCII digit anymore.

To fix this, we produce a mask that is all zeroes if any bit outside the
low byte is set in the code unit, all ones otherwise. Anding this mask
with the original code unit forces the table lookup to return the
sentinel value from the zero-index if any high bit was set and causes
the code unit not to be parsed as integer.

This bug was discovered when loading Mastodon posts inside the Ladybird
browser where some of Mastodon's JavaScript would trigger the code path
that erroneously parsed the emoji as integer. It had the visible effect
that some digits inside the posts would get rendered as one of the
emojis that parsed to that digit. For more details see this issue:
https://github.com/LadybirdBrowser/ladybird/issues/6205

The emojis in the test case are simply all the emojis used on Mastodon
that caused the bug. They can be found here:
06803422da/app/javascript/mastodon/features/emoji/emoji_map.json
This commit is contained in:
InvalidUsernameException 2025-09-15 23:05:25 +02:00
parent fec4082f01
commit 9d81c71aef
2 changed files with 277 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
include/fast_float/float_common.h
View File

@ -1132,7 +1132,13 @@ template <typename T> constexpr uint64_t int_luts<T>::min_safe_u64[];
template <typename UC> template <typename UC>
fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) { fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) {
return int_luts<>::chdigit[static_cast<unsigned char>(c)]; using UnsignedUC = typename std::make_unsigned<UC>::type;
auto uc = static_cast<UnsignedUC>(c);
// For types larger than one byte, we need to force an index with sentinel
// value (using index zero because that is easiest) if any byte other than
// the low byte is non-zero.
auto mask = static_cast<UnsignedUC>(-((uc & ~0xFFull) == 0));
return int_luts<>::chdigit[static_cast<unsigned char>(uc & mask)];
} }
fastfloat_really_inline constexpr size_t max_digits_u64(int base) { fastfloat_really_inline constexpr size_t max_digits_u64(int base) {

271
tests/fast_int.cpp
View File

@ -831,6 +831,275 @@ int main() {
return EXIT_FAILURE; return EXIT_FAILURE;
} }
} }
// dont parse UTF-16 code units of emojis as int if low byte is ascii digit
{
const std::u16string emojis[] = {
u"ℹ", u"ℹ️", u"☸", u"☸️", u"☹", u"☹️", u"✳", u"✳️",
u"✴", u"✴️", u"⤴", u"⤴️", u"⤵", u"⤵️", u"〰", u"〰️",
};
bool failed = false;
auto array_size = sizeof(emojis) / sizeof(emojis[0]);
for (size_t i = 0; i < array_size; i++) {
auto e = emojis[i];
int foo;
auto answer = fast_float::from_chars(e.data(), e.data() + e.size(), foo);
if (answer.ec == std::errc()) {
failed = true;
std::cerr << "Incorrectly parsed emoji #" << i << " as integer " << foo
<< "." << std::endl;
}
}
if (failed) {
return EXIT_FAILURE;
}
}
// dont parse UTF-32 code points of emojis as int if low byte is ascii digit
{
const std::u32string emojis[] = {
U"ℹ",
U"ℹ️",
U"☸",
U"☸️",
U"☹",
U"☹️",
U"✳",
U"✳️",
U"✴",
U"✴️",
U"⤴",
U"⤴️",
U"⤵",
U"⤵️",
U"〰",
U"〰️",
U"🈲",
U"🈳",
U"🈴",
U"🈵",
U"🈶",
U"🈷",
U"🈷️",
U"🈸",
U"🈹",
U"🌰",
U"🌱",
U"🌲",
U"🌳",
U"🌴",
U"🌵",
U"🌶",
U"🌶️",
U"🌷",
U"🌸",
U"🌹",
U"🐰",
U"🐱",
U"🐲",
U"🐳",
U"🐴",
U"🐵",
U"🐶",
U"🐷",
U"🐸",
U"🐹",
U"🔰",
U"🔱",
U"🔲",
U"🔳",
U"🔴",
U"🔵",
U"🔶",
U"🔷",
U"🔸",
U"🔹",
U"😰",
U"😱",
U"😲",
U"😳",
U"😴",
U"😵",
U"😵‍💫",
U"😶",
U"😶‍🌫",
U"😶‍🌫️",
U"😷",
U"😸",
U"😹",
U"🤰",
U"🤰🏻",
U"🤰🏼",
U"🤰🏽",
U"🤰🏾",
U"🤰🏿",
U"🤱",
U"🤱🏻",
U"🤱🏼",
U"🤱🏽",
U"🤱🏾",
U"🤱🏿",
U"🤲",
U"🤲🏻",
U"🤲🏼",
U"🤲🏽",
U"🤲🏾",
U"🤲🏿",
U"🤳",
U"🤳🏻",
U"🤳🏼",
U"🤳🏽",
U"🤳🏾",
U"🤳🏿",
U"🤴",
U"🤴🏻",
U"🤴🏼",
U"🤴🏽",
U"🤴🏾",
U"🤴🏿",
U"🤵",
U"🤵‍♀",
U"🤵‍♀️",
U"🤵‍♂",
U"🤵‍♂️",
U"🤵🏻",
U"🤵🏻‍♀",
U"🤵🏻‍♀️",
U"🤵🏻‍♂",
U"🤵🏻‍♂️",
U"🤵🏼",
U"🤵🏼‍♀",
U"🤵🏼‍♀️",
U"🤵🏼‍♂",
U"🤵🏼‍♂️",
U"🤵🏽",
U"🤵🏽‍♀",
U"🤵🏽‍♀️",
U"🤵🏽‍♂",
U"🤵🏽‍♂️",
U"🤵🏾",
U"🤵🏾‍♀",
U"🤵🏾‍♀️",
U"🤵🏾‍♂",
U"🤵🏾‍♂️",
U"🤵🏿",
U"🤵🏿‍♀",
U"🤵🏿‍♀️",
U"🤵🏿‍♂",
U"🤵🏿‍♂️",
U"🤶",
U"🤶🏻",
U"🤶🏼",
U"🤶🏽",
U"🤶🏾",
U"🤶🏿",
U"🤷",
U"🤷‍♀",
U"🤷‍♀️",
U"🤷‍♂",
U"🤷‍♂️",
U"🤷🏻",
U"🤷🏻‍♀",
U"🤷🏻‍♀️",
U"🤷🏻‍♂",
U"🤷🏻‍♂️",
U"🤷🏼",
U"🤷🏼‍♀",
U"🤷🏼‍♀️",
U"🤷🏼‍♂",
U"🤷🏼‍♂️",
U"🤷🏽",
U"🤷🏽‍♀",
U"🤷🏽‍♀️",
U"🤷🏽‍♂",
U"🤷🏽‍♂️",
U"🤷🏾",
U"🤷🏾‍♀",
U"🤷🏾‍♀️",
U"🤷🏾‍♂",
U"🤷🏾‍♂️",
U"🤷🏿",
U"🤷🏿‍♀",
U"🤷🏿‍♀️",
U"🤷🏿‍♂",
U"🤷🏿‍♂️",
U"🤸",
U"🤸‍♀",
U"🤸‍♀️",
U"🤸‍♂",
U"🤸‍♂️",
U"🤸🏻",
U"🤸🏻‍♀",
U"🤸🏻‍♀️",
U"🤸🏻‍♂",
U"🤸🏻‍♂️",
U"🤸🏼",
U"🤸🏼‍♀",
U"🤸🏼‍♀️",
U"🤸🏼‍♂",
U"🤸🏼‍♂️",
U"🤸🏽",
U"🤸🏽‍♀",
U"🤸🏽‍♀️",
U"🤸🏽‍♂",
U"🤸🏽‍♂️",
U"🤸🏾",
U"🤸🏾‍♀",
U"🤸🏾‍♀️",
U"🤸🏾‍♂",
U"🤸🏾‍♂️",
U"🤸🏿",
U"🤸🏿‍♀",
U"🤸🏿‍♀️",
U"🤸🏿‍♂",
U"🤸🏿‍♂️",
U"🤹",
U"🤹‍♀",
U"🤹‍♀️",
U"🤹‍♂",
U"🤹‍♂️",
U"🤹🏻",
U"🤹🏻‍♀",
U"🤹🏻‍♀️",
U"🤹🏻‍♂",
U"🤹🏻‍♂️",
U"🤹🏼",
U"🤹🏼‍♀",
U"🤹🏼‍♀️",
U"🤹🏼‍♂",
U"🤹🏼‍♂️",
U"🤹🏽",
U"🤹🏽‍♀",
U"🤹🏽‍♀️",
U"🤹🏽‍♂",
U"🤹🏽‍♂️",
U"🤹🏾",
U"🤹🏾‍♀",
U"🤹🏾‍♀️",
U"🤹🏾‍♂",
U"🤹🏾‍♂️",
U"🤹🏿",
U"🤹🏿‍♀",
U"🤹🏿‍♀️",
U"🤹🏿‍♂",
U"🤹🏿‍♂️",
};
bool failed = false;
auto array_size = sizeof(emojis) / sizeof(emojis[0]);
for (size_t i = 0; i < array_size; i++) {
auto e = emojis[i];
int foo;
auto answer = fast_float::from_chars(e.data(), e.data() + e.size(), foo);
if (answer.ec == std::errc()) {
failed = true;
std::cerr << "Incorrectly parsed emoji #" << i << " as integer " << foo
<< "." << std::endl;
}
}
if (failed) {
return EXIT_FAILURE;
}
}
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
@ -842,4 +1111,4 @@ int main() {
std::cerr << "The test requires C++17." << std::endl; std::cerr << "The test requires C++17." << std::endl;
return EXIT_SUCCESS; return EXIT_SUCCESS;
} }
#endif #endif