coffee/fast_float
1
0
Fork 0
mirror of https://github.com/fastfloat/fast_float.git synced 2026-10-31 03:16:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge 784cd670c3bb50162ed8ccdd17e7bc9a43af2b90 into 0de7a21da582b6c84b0546772a96549882905d2b

Browse Source
This commit is contained in:
Raúl Marín 2026-10-19 23:01:22 +00:00 committed by GitHub
commit da1a459e74
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 180 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
include/fast_float/ascii_number.h
View File

@ -116,6 +116,34 @@ fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) {
FASTFLOAT_SIMD_RESTORE_WARNINGS
}
// i followed by eight digit units, combining digit pairs in the vector unit
// (pmaddwd by 10,1 then 100,1): parse_eight_digits_unrolled on the packed
// bytes needs three 64-bit constants in general registers, and GCC 14/15
// spill a value on the digit chain to make room.
fastfloat_really_inline uint64_t simd_append_eight_digits(uint64_t i,
__m128i const data) {
FASTFLOAT_SIMD_DISABLE_WARNINGS
// On the characters rather than the digits, with '0' * 11111111 taken off
// at the end: one step less on the digit chain (pairs stay below 628 and
// groups of four below 63328, in range for pmaddwd).
__m128i const pairs =
_mm_madd_epi16(data, _mm_set_epi16(1, 10, 1, 10, 1, 10, 1, 10));
__m128i const quads =
_mm_madd_epi16(_mm_packs_epi32(pairs, pairs),
_mm_set_epi16(1, 100, 1, 100, 1, 100, 1, 100));
#ifdef FASTFLOAT_64BIT
uint64_t const both = uint64_t(_mm_cvtsi128_si64(quads));
#else
uint64_t both;
_mm_storel_epi64(reinterpret_cast<__m128i *>(&both), quads);
#endif
// The offset goes on the i side, which is ready early (clang otherwise
// subtracts it on the digit chain).
return (i * 100000000 - uint64_t('0') * 11111111) +
(uint32_t(both) * 10000 + uint32_t(both >> 32));
FASTFLOAT_SIMD_RESTORE_WARNINGS
}
#elif defined(FASTFLOAT_NEON)
fastfloat_really_inline uint64_t simd_read8_to_u64(uint16x8_t const data) {
@ -132,6 +160,23 @@ fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) {
FASTFLOAT_SIMD_RESTORE_WARNINGS
}
// See the SSE2 version.
fastfloat_really_inline uint64_t
simd_append_eight_digits(uint64_t i, uint16x8_t const data) {
FASTFLOAT_SIMD_DISABLE_WARNINGS
static uint16_t const m10[8] = {10, 1, 10, 1, 10, 1, 10, 1};
static uint16_t const m100[4] = {100, 1, 100, 1};
uint32x4_t const pairs = vpaddlq_u16(vmulq_u16(data, vld1q_u16(m10)));
uint32x2_t const quads =
vpaddl_u16(vmul_u16(vmovn_u32(pairs), vld1_u16(m100)));
uint64_t const both = vget_lane_u64(vreinterpret_u64_u32(quads), 0);
// The offset goes on the i side, which is ready early (clang otherwise
// subtracts it on the digit chain).
return (i * 100000000 - uint64_t('0') * 11111111) +
(uint32_t(both) * 10000 + uint32_t(both >> 32));
FASTFLOAT_SIMD_RESTORE_WARNINGS
}
#endif // FASTFLOAT_SSE2
// MSVC SFINAE is broken pre-VS2017
@ -208,7 +253,7 @@ simd_parse_if_eight_digits_unrolled(char16_t const *chars,
__m128i const t1 = _mm_cmpgt_epi16(t0, _mm_set1_epi16(-32759));
if (_mm_movemask_epi8(t1) == 0) {
i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data));
i = simd_append_eight_digits(i, data);
return true;
} else
return false;
@ -223,7 +268,7 @@ simd_parse_if_eight_digits_unrolled(char16_t const *chars,
uint16x8_t const mask = vcltq_u16(t0, vmovq_n_u16('9' - '0' + 1));
if (vminvq_u16(mask) == 0xFFFF) {
i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data));
i = simd_append_eight_digits(i, data);
return true;
} else
return false;
@ -235,6 +280,38 @@ simd_parse_if_eight_digits_unrolled(char16_t const *chars,
#endif // FASTFLOAT_SSE2
}
// Four units, for a remaining 4-7 digit run: the same test as above on a
// 64-bit load, so the constants stay in vector registers or memory.
fastfloat_really_inline bool simd_parse_if_four_digits(char16_t const *chars,
uint64_t &i) noexcept {
#ifdef FASTFLOAT_SSE2
FASTFLOAT_SIMD_DISABLE_WARNINGS
__m128i const data =
_mm_loadl_epi64(reinterpret_cast<__m128i const *>(chars));
__m128i const t0 = _mm_add_epi16(data, _mm_set1_epi16(32720));
__m128i const t1 = _mm_cmpgt_epi16(t0, _mm_set1_epi16(-32759));
if ((_mm_movemask_epi8(t1) & 0xFF) != 0) {
return false;
}
uint32_t const val =
uint32_t(_mm_cvtsi128_si32(_mm_packus_epi16(data, data)));
FASTFLOAT_SIMD_RESTORE_WARNINGS
#elif defined(FASTFLOAT_NEON)
FASTFLOAT_SIMD_DISABLE_WARNINGS
uint16x4_t const data = vld1_u16(reinterpret_cast<uint16_t const *>(chars));
uint16x4_t const t0 = vsub_u16(data, vdup_n_u16('0'));
uint16x4_t const mask = vclt_u16(t0, vdup_n_u16('9' - '0' + 1));
if (vget_lane_u64(vreinterpret_u64_u16(mask), 0) != ~uint64_t(0)) {
return false;
}
uint32_t const val = vget_lane_u32(
vreinterpret_u32_u8(vmovn_u16(vcombine_u16(data, data))), 0);
FASTFLOAT_SIMD_RESTORE_WARNINGS
#endif
i = i * 10000 + parse_four_digits_unrolled(val);
return true;
}
#endif // FASTFLOAT_HAS_SIMD
// MSVC SFINAE is broken pre-VS2017
@ -248,6 +325,16 @@ bool simd_parse_if_eight_digits_unrolled(UC const *, uint64_t &) {
return 0;
}
#if defined(_MSC_VER) && _MSC_VER <= 1900
template <typename UC>
#else
template <typename UC, FASTFLOAT_ENABLE_IF(!has_simd_opt<UC>()) = 0>
#endif
// dummy for compile
bool simd_parse_if_four_digits(UC const *, uint64_t &) {
return false;
}
template <typename UC, FASTFLOAT_ENABLE_IF(!std::is_same<UC, char>::value) = 0>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
loop_parse_if_eight_digits(UC const *&p, UC const *const pend, uint64_t &i) {
@ -259,6 +346,12 @@ loop_parse_if_eight_digits(UC const *&p, UC const *const pend, uint64_t &i) {
p, i)) { // in rare cases, this will overflow, but that's ok
p += 8;
}
// A remaining 4-7 digit run in one step, as for char, so fewer digits go
// through the serial scalar loop.
if (!cpp20_and_in_constexpr() && (pend - p) >= 4 &&
simd_parse_if_four_digits(p, i)) {
p += 4;
}
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
@ -391,27 +484,32 @@ parse_number_string_impl(UC const *p, UC const *pend,
// Straight-line unroll of the integer-part scan: most integer parts are
// 1-5 digits, so peeling the first iterations eliminates the loop back-edge
// for the common case. Semantics are identical to the original `while` loop:
// i = 10*i + digit, advancing p.
if ((p != pend) && is_integer(*p)) {
i = uint64_t(*p - UC('0'));
// i = 10*i + int_digit, advancing p. Each int_digit is tested and used
// through one subtraction, as in the fraction tail (clang otherwise computes
// it twice).
uint64_t int_digit;
if ((p != pend) && (int_digit = uint64_t(*p) - uint64_t(UC('0'))) <= 9) {
i = int_digit;
++p;
if ((p != pend) && is_integer(*p)) {
i = 10 * i + uint64_t(*p - UC('0'));
if ((p != pend) && (int_digit = uint64_t(*p) - uint64_t(UC('0'))) <= 9) {
i = 10 * i + int_digit;
++p;
if ((p != pend) && is_integer(*p)) {
i = 10 * i + uint64_t(*p - UC('0'));
if ((p != pend) && (int_digit = uint64_t(*p) - uint64_t(UC('0'))) <= 9) {
i = 10 * i + int_digit;
++p;
if ((p != pend) && is_integer(*p)) {
i = 10 * i + uint64_t(*p - UC('0'));
if ((p != pend) &&
(int_digit = uint64_t(*p) - uint64_t(UC('0'))) <= 9) {
i = 10 * i + int_digit;
++p;
if ((p != pend) && is_integer(*p)) {
i = 10 * i + uint64_t(*p - UC('0'));
if ((p != pend) &&
(int_digit = uint64_t(*p) - uint64_t(UC('0'))) <= 9) {
i = 10 * i + int_digit;
++p;
while ((p != pend) && is_integer(*p)) {
while ((p != pend) &&
(int_digit = uint64_t(*p) - uint64_t(UC('0'))) <= 9) {
// a multiplication by 10 is cheaper than an arbitrary integer
// multiplication
i = 10 * i +
uint64_t(*p - UC('0')); // might overflow, handled later
i = 10 * i + int_digit; // might overflow, handled later
++p;
}
}
@ -468,8 +566,13 @@ parse_number_string_impl(UC const *p, UC const *pend,
// for integers with many digits, digit parsing is the primary bottleneck.
loop_parse_if_eight_digits(p, pend, i);
while ((p != pend) && is_integer(*p)) {
uint8_t digit = uint8_t(*p - UC('0'));
// One subtraction serves the digit test and the digit (GCC computes it
// twice when written separately); in 64 bits it needs no extension.
while (p != pend) {
uint64_t const digit = uint64_t(*p) - uint64_t(UC('0'));
if (digit > 9) {
break;
}
++p;
i = i * 10 + digit; // in rare cases, this will overflow, but that's ok
}
@ -500,14 +603,13 @@ parse_number_string_impl(UC const *p, UC const *pend,
(UC('D') == *p)) {
++p;
}
// No branch on the sign: it is as unpredictable as the value (`|`, not
// `||`, which GCC turns into a branch).
bool neg_exp = false;
if ((p != pend) && (UC('-') == *p)) {
neg_exp = true;
++p;
} else if ((p != pend) &&
(UC('+') ==
*p)) { // '+' on exponent is allowed by C++17 20.19.3.(7.1)
++p;
if (p != pend) {
neg_exp = (UC('-') == *p);
// '+' on exponent is allowed by C++17 20.19.3.(7.1)
p += (neg_exp | (UC('+') == *p));
}
if ((p == pend) || !is_integer(*p)) {
if (!uint64_t(fmt & chars_format::fixed)) {
@ -519,13 +621,28 @@ parse_number_string_impl(UC const *p, UC const *pend,
// Otherwise, we will be ignoring the 'e'.
p = location_of_e;
} else {
while ((p != pend) && is_integer(*p)) {
uint8_t digit = uint8_t(*p - UC('0'));
if (exp_number < 0x10000000) {
exp_number = 10 * exp_number + digit;
// Same digit test as the fraction tail. No saturation in the loop: it
// would add a select per digit on the path to the power of ten, and 18
// digits fit in int64_t.
UC const *const exp_start = p;
uint64_t exp_digits = 0;
while (p != pend) {
uint64_t const digit = uint64_t(*p) - uint64_t(UC('0'));
if (digit > 9) {
break;
}
exp_digits = 10 * exp_digits + digit;
++p;
}
exp_number = int64_t(exp_digits);
if fastfloat_unlikely (p - exp_start > 18) {
exp_number = 0;
for (UC const *q = exp_start; q != p; ++q) {
if (exp_number < 0x10000000) {
exp_number = 10 * exp_number + (*q - UC('0'));
}
}
}
if (neg_exp) {
exp_number = -exp_number;
}

21
include/fast_float/decimal_to_binary.h
View File

@ -104,19 +104,20 @@ compute_error(int64_t q, uint64_t w) noexcept {
// packed. However, in some very rare cases, the computation will fail. In such
// cases, we return an adjusted_mantissa with a negative power of 2: the caller
// should recompute in such cases.
template <typename binary>
// known_in_range: the caller knows that q is within the range of powers of
// ten, so only w == 0 is checked.
template <typename binary, bool known_in_range = false>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
compute_float(int64_t q, uint64_t w) noexcept {
adjusted_mantissa answer;
if ((w == 0) || (q < binary::smallest_power_of_ten())) {
answer.power2 = 0;
answer.mantissa = 0;
// result should be zero
return answer;
}
if (q > binary::largest_power_of_ten()) {
// we want to get infinity:
answer.power2 = binary::infinite_power();
// One comparison for both ends of the range, then zero vs infinity without
// a branch: it follows the sign of the exponent, which is unpredictable.
bool const q_out_of_range = uint64_t(q - binary::smallest_power_of_ten()) >
uint64_t(binary::largest_power_of_ten() -
binary::smallest_power_of_ten());
if ((w == 0) || (!known_in_range && q_out_of_range)) {
bool const underflow = (w == 0) | (q < binary::smallest_power_of_ten());
answer.power2 = int32_t(!underflow) * binary::infinite_power();
answer.mantissa = 0;
return answer;
}

33
include/fast_float/parse_number.h
View File

@ -193,7 +193,7 @@ template <> struct from_chars_caller<std::float64_t> {
// whole parser into each caller, where the format is a constant; clang keeps
// it out of line and would otherwise re-test each format flag per conversion.
template <typename T, typename UC, chars_format Fmt>
FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
fastfloat_clang_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
from_chars_fixed_format(UC const *first, UC const *last, T &value) noexcept {
return from_chars_caller<T>::call(first, last, value,
parse_options_t<UC>(Fmt));
@ -201,7 +201,7 @@ from_chars_fixed_format(UC const *first, UC const *last, T &value) noexcept {
#endif
template <typename T, typename UC, typename>
FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
fastfloat_clang_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
from_chars(UC const *first, UC const *last, T &value,
chars_format fmt /*= chars_format::general*/) noexcept {
#ifdef __clang__
@ -451,8 +451,20 @@ from_chars_float_advanced(UC const *first, UC const *last, T &value,
return answer;
}
#ifdef __clang__
// If the exponent passed Clinger's range test, it is well within range. GCC
// carries this into compute_float by itself; clang needs the instantiation
// without the range check. The mantissa can still be zero when the rounding
// mode is not to nearest.
adjusted_mantissa am =
(binary_format<T>::min_exponent_fast_path() <= pns.exponent &&
pns.exponent <= binary_format<T>::max_exponent_fast_path())
? compute_float<binary_format<T>, true>(pns.exponent, pns.mantissa)
: compute_float<binary_format<T>>(pns.exponent, pns.mantissa);
#else
adjusted_mantissa am =
compute_float<binary_format<T>>(pns.exponent, pns.mantissa);
#endif
// Slow path B (rare): Eisel-Lemire could not resolve; digit_comp needs the
// integer/fraction spans. Route to the cold helper (clinger there is a
// dead-effect since it already failed here; the cold re-parse + digit_comp
@ -461,14 +473,15 @@ from_chars_float_advanced(UC const *first, UC const *last, T &value,
return parse_number_slow_path<T, UC>(first, last, value, options, bjf);
}
to_float(pns.negative, am, value);
// Test for over/underflow. Marked unlikely so that clang keeps it as a
// branch instead of folding it into a chain of conditional moves that
// every conversion pays for.
if fastfloat_clang_unlikely ((pns.mantissa != 0 && am.mantissa == 0 &&
am.power2 == 0) ||
am.power2 ==
binary_format<T>::infinite_power()) {
answer.ec = std::errc::result_out_of_range;
// Test for over/underflow. One comparison catches both a zero and an
// infinite exponent, so normal values pay a single, never-taken branch. The
// result is then selected: which case it is follows the sign of the exponent.
constexpr uint32_t max_power2 =
uint32_t(binary_format<T>::infinite_power() - 1);
if fastfloat_clang_unlikely (uint32_t(am.power2 - 1) >= max_power2) {
bool const out_of_range =
(am.power2 != 0) | ((am.mantissa == 0) & (pns.mantissa != 0));
answer.ec = out_of_range ? std::errc::result_out_of_range : answer.ec;
}
#ifdef __clang__
#pragma clang diagnostic pop