diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index 1c362be..df9e405 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -19,7 +19,7 @@ namespace fast_float { template -fastfloat_really_inline FASTFLOAT_CONSTEVAL bool has_simd_opt() noexcept { +fastfloat_inline FASTFLOAT_CONSTEVAL bool has_simd_opt() noexcept { #ifdef FASTFLOAT_USE_SIMD return std::is_same::value; #else @@ -30,7 +30,7 @@ fastfloat_really_inline FASTFLOAT_CONSTEVAL bool has_simd_opt() noexcept { // Next function can be micro-optimized, but compilers are entirely // able to optimize it well. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 bool is_integer(UC c) noexcept { +fastfloat_inline FASTFLOAT_CONSTEXPR14 bool is_integer(UC c) noexcept { const auto d = c - UC('0'); // UC may be signed. return d >= 0 && d <= 9; @@ -44,14 +44,14 @@ using std::byteswap; } #else -fastfloat_really_inline constexpr uint64_t byteswap(uint64_t val) noexcept { +fastfloat_inline constexpr uint64_t byteswap(uint64_t val) noexcept { return (val & 0xFF00000000000000) >> 56 | (val & 0x00FF000000000000) >> 40 | (val & 0x0000FF0000000000) >> 24 | (val & 0x000000FF00000000) >> 8 | (val & 0x00000000FF000000) << 8 | (val & 0x0000000000FF0000) << 24 | (val & 0x000000000000FF00) << 40 | (val & 0x00000000000000FF) << 56; } -fastfloat_really_inline constexpr uint32_t byteswap(uint32_t val) noexcept { +fastfloat_inline constexpr uint32_t byteswap(uint32_t val) noexcept { return (val >> 24) | ((val >> 8) & 0x0000FF00u) | ((val << 8) & 0x00FF0000u) | (val << 24); } @@ -62,7 +62,7 @@ fastfloat_really_inline constexpr uint32_t byteswap(uint32_t val) noexcept { // Read UCs into an unsigned integer. Truncates UC if not char. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 T +fastfloat_inline FASTFLOAT_CONSTEXPR20 T read_chars_to_unsigned(UC const *chars) noexcept { if (is_constant_evaluated() || !std::is_same::value) { T val = 0; @@ -85,7 +85,7 @@ read_chars_to_unsigned(UC const *chars) noexcept { #if FASTFLOAT_X86_SIMD -fastfloat_really_inline uint64_t simd_read8(__m128i const data) { +fastfloat_inline uint64_t simd_read8(__m128i const data) { // _mm_packus_epi16 is SSE2, converts 8×u16 → 8×u8 __m128i const packed = _mm_packus_epi16(data, data); @@ -104,7 +104,7 @@ fastfloat_really_inline uint64_t simd_read8(__m128i const data) { #endif } -fastfloat_really_inline uint64_t simd_read8(char16_t const *chars) { +fastfloat_inline uint64_t simd_read8(char16_t const *chars) { FASTFLOAT_SIMD_DISABLE_WARNINGS // unaligned SIMD instruction -> all fine. return simd_read8(_mm_loadu_si128(reinterpret_cast<__m128i const *>(chars))); @@ -113,12 +113,12 @@ fastfloat_really_inline uint64_t simd_read8(char16_t const *chars) { #elif FASTFLOAT_ARM_NEON -fastfloat_really_inline uint64_t simd_read8(uint16x8_t const &data) { +fastfloat_inline uint64_t simd_read8(uint16x8_t const &data) { uint8x8_t utf8_packed = vmovn_u16(data); return vget_lane_u64(vreinterpret_u64_u8(utf8_packed), 0); } -fastfloat_really_inline uint64_t simd_read8(char16_t const *chars) { +fastfloat_inline uint64_t simd_read8(char16_t const *chars) { FASTFLOAT_SIMD_DISABLE_WARNINGS return simd_read8(vld1q_u16(reinterpret_cast(chars))); FASTFLOAT_SIMD_RESTORE_WARNINGS @@ -140,7 +140,7 @@ uint64_t simd_read8(UC const *) { } // credit @aqrit -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint32_t +fastfloat_inline FASTFLOAT_CONSTEXPR14 uint32_t parse_8_digits(uint64_t val) noexcept { uint64_t const mask = 0x000000FF000000FF; uint64_t const mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32) @@ -153,7 +153,7 @@ parse_8_digits(uint64_t val) noexcept { // Call this if chars are definitely 8 digits. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint32_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 uint32_t parse_8_digits(UC const *chars) noexcept { if (is_constant_evaluated() || !has_simd_opt()) { return parse_8_digits( @@ -163,18 +163,16 @@ parse_8_digits(UC const *chars) noexcept { } // credit @aqrit -fastfloat_really_inline constexpr bool -is_made_of_8_digits(uint64_t val) noexcept { +fastfloat_inline constexpr bool is_made_of_8_digits(uint64_t val) noexcept { return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) & 0x8080808080808080)); } -fastfloat_really_inline constexpr bool -is_made_of_4_digits(uint32_t val) noexcept { +fastfloat_inline constexpr bool is_made_of_4_digits(uint32_t val) noexcept { return !((((val + 0x46464646) | (val - 0x30303030)) & 0x80808080)); } -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint32_t +fastfloat_inline FASTFLOAT_CONSTEXPR14 uint32_t parse_4_digits(uint32_t val) noexcept { val -= 0x30303030; val = (val * 10) + (val >> 8); @@ -187,7 +185,7 @@ parse_4_digits(uint32_t val) noexcept { #if FASTFLOAT_X86_SIMD >= 31 // credit @hedgehoginthecpp -fastfloat_really_inline __m128i parse_4x4_digits(__m128i data) noexcept { +fastfloat_inline __m128i parse_4x4_digits(__m128i data) noexcept { // 1. convert from ASCII '0' .. '9' to numbers 0 .. 9 const __m128i ascii0 = _mm_set1_epi8('0'); const __m128i t0 = _mm_subs_epu8(data, ascii0); @@ -204,8 +202,7 @@ fastfloat_really_inline __m128i parse_4x4_digits(__m128i data) noexcept { #endif // credit @hedgehoginthecpp -fastfloat_really_inline uint64_t -convert_4x4_to_16_digits(__m128i data) noexcept { +fastfloat_inline uint64_t convert_4x4_to_16_digits(__m128i data) noexcept { // 4. convert to 16-digit number // v[0] * 10^12 + v[1] * 10^8 + v[2] * 10^4 + v[3] const uint64_t lo = static_cast(_mm_cvtsi128_si32(data)); @@ -222,8 +219,8 @@ convert_4x4_to_16_digits(__m128i data) noexcept { #if FASTFLOAT_X86_SIMD >= 42 // credit @hedgehoginthecpp -fastfloat_really_inline bool parse_if_16_digits(char const *chars, - uint64_t &value) noexcept { +fastfloat_inline bool parse_if_16_digits(char const *chars, + uint64_t &value) noexcept { FASTFLOAT_SIMD_DISABLE_WARNINGS const __m128i data = _mm_loadu_si128(reinterpret_cast<__m128i const *>(chars)); @@ -262,14 +259,14 @@ fastfloat_really_inline bool parse_if_16_digits(char const *chars, #if FASTFLOAT_X86_SIMD >= 31 // credit @hedgehoginthecpp -fastfloat_really_inline uint64_t parse_16_digits(char const *p) noexcept { +fastfloat_inline uint64_t parse_16_digits(char const *p) noexcept { const __m128i data = _mm_loadu_si128(reinterpret_cast(p)); return convert_4x4_to_16_digits(parse_4x4_digits(data)); } #endif // credit @hedgehoginthecpp -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void +fastfloat_inline FASTFLOAT_CONSTEXPR20 void parse_digits_until_19(char const *&p, char const *pend, am_mant_t &mantissa) { #if FASTFLOAT_X86_SIMD >= 31 if (!is_constant_evaluated()) { @@ -302,7 +299,7 @@ parse_digits_until_19(char const *&p, char const *pend, am_mant_t &mantissa) { } template ::value) = 0> -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void +fastfloat_inline FASTFLOAT_CONSTEXPR20 void parse_digits_until_19(UC const *&p, UC const *pend, am_mant_t &mantissa) noexcept { do { @@ -311,7 +308,7 @@ parse_digits_until_19(UC const *&p, UC const *pend, } #else template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void +fastfloat_inline FASTFLOAT_CONSTEXPR20 void parse_digits_until_19(UC const *&p, UC const *pend, am_mant_t &mantissa) noexcept { do { @@ -323,7 +320,7 @@ parse_digits_until_19(UC const *&p, UC const *pend, // Call this if chars might not be 8 digits. // Using this style (instead of is_made_of_8_digits() then // parse_8_digits()) ensures we don't load SIMD registers twice. -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool simd_parse_if_8_digits(char16_t const *chars, uint64_t &i) noexcept { if (is_constant_evaluated()) { return false; @@ -370,7 +367,7 @@ simd_parse_if_8_digits(char16_t const *chars, uint64_t &i) noexcept { } #else template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void +fastfloat_inline FASTFLOAT_CONSTEXPR20 void parse_digits_until_19(UC const *&p, UC const *pend, am_mant_t &mantissa) noexcept { do { @@ -391,7 +388,7 @@ constexpr bool simd_parse_if_8_digits(UC const *, uint64_t &) { } template ::value) = 0> -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void +fastfloat_inline FASTFLOAT_CONSTEXPR20 void loop_parse_if_digits(UC const *&p, UC const *const pend, uint64_t &i) noexcept { if (!is_constant_evaluated()) { if FASTFLOAT_CONSTEXPR17 (has_simd_opt()) { @@ -408,7 +405,7 @@ loop_parse_if_digits(UC const *&p, UC const *const pend, uint64_t &i) noexcept { } } -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void +fastfloat_inline FASTFLOAT_CONSTEXPR20 void loop_parse_if_digits(char const *&p, char const *const pend, uint64_t &i) noexcept { #if FASTFLOAT_USE_SIMD && FASTFLOAT_X86_SIMD >= 42 @@ -499,7 +496,7 @@ using parsed_number_string = parsed_number_string_t; // Helper for error creating template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t & +fastfloat_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t & report_parse_error(parsed_number_string_t &answer, UC const *p, parse_error error) noexcept { answer.invalid = true; @@ -517,11 +514,11 @@ report_parse_error(parsed_number_string_t &answer, UC const *p, // spans (read only by the rare digit_comp slow path) are not materialized, // which keeps the fat parsed_number_string_t off the hot path. The caller // re-parses with store_spans=true if the slow path is actually reached. -template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t -parse_number_string(UC const *p, UC const *pend, - parse_options_t const options, - bool store_spans = true) noexcept { +template +fastfloat_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t +parse_number_string_impl(UC const *p, UC const *pend, + parse_options_t const options, + bool const store_spans) noexcept { parsed_number_string_t answer{}; FASTFLOAT_ASSUME(p < pend); // so dereference without checks #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN @@ -555,7 +552,7 @@ parse_number_string(UC const *p, UC const *pend, // HedgehogInTheCPP: compiler generate much better code in any mode without // manual unroling because it's less branching and also it's allow more - // inlining which is better. A a multiplication by 10 is cheaper than an + // inlining which is better. A multiplication by 10 is cheaper than an // arbitrary integer multiplication. might overflow, handled later #if !defined(FASTFLOAT_ISNOT_CHECKED_BOUNDS) && \ !defined(FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN) @@ -584,7 +581,7 @@ parse_number_string(UC const *p, UC const *pend, return report_parse_error(answer, p, parse_error::no_digits_in_integer_part); } - if (json_fmt || chars_format_t(options.format & detail::javascript_fmt)) { + if (json_fmt || javascript_fmt) { // ECMAScript DecimalIntegerLiteral is "0" or a non-zero digit followed by // digits: no leading zeros. Unlike JSON, the integer part may be empty // (".5"); the no_digits_in_mantissa check below still rejects ".". @@ -595,7 +592,7 @@ parse_number_string(UC const *p, UC const *pend, return report_parse_error( answer, start_digits, parse_error::leading_zeros_in_integer_part); } - if (!json_fmt) { + if (javascript_fmt) { // Sloppy mode (Annex B): a NonOctalDecimalIntegerLiteral has a leading // zero and at least one digit that is 8 or 9 ("08.5" is 8.5). With // octal digits only, it is a LegacyOctalIntegerLiteral ("0775"), which @@ -765,8 +762,45 @@ parse_number_string(UC const *p, UC const *pend, return answer; } +#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN +// Cold instantiation of the parser: the ECMAScript integer-part rule costs two +// error returns plus the Annex B octal scan, and parse_number_string_impl is +// force-inlined, so an inlined javascript body would enlarge the frame of every +// caller that never asks for it. Out of line, it costs those callers nothing. +template +fastfloat_noinline FASTFLOAT_CONSTEXPR20 parsed_number_string_t +parse_number_string_javascript(UC const *p, UC const *pend, + parse_options_t const options, + bool const store_spans) noexcept { + return parse_number_string_impl(p, pend, options, + store_spans); +} + +// Public entry point, behaviour unchanged: chars_format::javascript is still +// honoured, it is just selected here once instead of being re-tested inside the +// parser loop. Callers that have already ruled the format out (see +// from_chars_float_advanced) should call parse_number_string_impl directly so +// that not even this test reaches their hot path. +template +fastfloat_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t +parse_number_string(UC const *p, UC const *pend, + parse_options_t const options, + bool const store_spans = true) noexcept { + // JSON and JavaScript are mutually exclusive, so only the non-JSON + // instantiation has to look at the flag. + if (!basic_json_fmt) { + if fastfloat_unlikely (chars_format_t(options.format & + detail::basic_javascript_fmt)) { + return parse_number_string_javascript(p, pend, options, store_spans); + } + } + return parse_number_string_impl(p, pend, options, + store_spans); +} +#endif + template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t parse_int_string(UC const *p, UC const *pend, T &value, parse_options_t const options) noexcept { FASTFLOAT_ASSUME(p < pend); // so dereference without checks diff --git a/include/fast_float/bigint.h b/include/fast_float/bigint.h index e436d58..9d126b9 100644 --- a/include/fast_float/bigint.h +++ b/include/fast_float/bigint.h @@ -158,20 +158,20 @@ template struct stackvec { } }; -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint64_t +fastfloat_inline FASTFLOAT_CONSTEXPR14 uint64_t empty_hi64(bool &truncated) noexcept { truncated = false; return 0; } -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t uint64_hi64(uint64_t r0, bool &truncated) noexcept { truncated = false; auto const shl = leading_zeroes(r0); return r0 << shl; } -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t uint64_hi64(uint64_t r0, uint64_t r1, bool &truncated) noexcept { auto const shl = leading_zeroes(r0); if (shl == 0) { @@ -184,19 +184,19 @@ uint64_hi64(uint64_t r0, uint64_t r1, bool &truncated) noexcept { } } -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t uint32_hi64(uint32_t r0, bool &truncated) noexcept { return uint64_hi64(r0, truncated); } -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t uint32_hi64(uint32_t r0, uint32_t r1, bool &truncated) noexcept { uint64_t x0 = r0; uint64_t x1 = r1; return uint64_hi64((x0 << 32) | x1, truncated); } -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool &truncated) noexcept { uint64_t x0 = r0; uint64_t x1 = r1; @@ -208,7 +208,7 @@ uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool &truncated) noexcept { // we want an efficient operation. for msvc, where // we don't have built-in intrinsics, this is still // pretty fast. -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb +fastfloat_inline FASTFLOAT_CONSTEXPR20 limb scalar_add(limb x, limb y, bool &overflow) noexcept { limb z; // gcc and clang @@ -226,8 +226,8 @@ scalar_add(limb x, limb y, bool &overflow) noexcept { } // multiply two small integers, getting both the high and low bits. -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb -scalar_mul(limb x, limb y, limb &carry) noexcept { +fastfloat_inline FASTFLOAT_CONSTEXPR20 limb scalar_mul(limb x, limb y, + limb &carry) noexcept { #ifdef FASTFLOAT_64BIT_LIMB #if defined(__SIZEOF_INT128__) // GCC and clang both define it as an extension. @@ -254,7 +254,7 @@ scalar_mul(limb x, limb y, limb &carry) noexcept { // add scalar value to bigint starting from offset. // used in grade school multiplication template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool small_add_from(stackvec &vec, limb carry, limb_t start) noexcept { bool overflow; while (carry != 0 && start < vec.len()) { @@ -270,15 +270,15 @@ small_add_from(stackvec &vec, limb carry, limb_t start) noexcept { // add scalar value to bigint. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool -small_add(stackvec &vec, limb y) noexcept { +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool small_add(stackvec &vec, + limb y) noexcept { return small_add_from(vec, y, 0); } // multiply bigint by scalar value. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool -small_mul(stackvec &vec, limb y) noexcept { +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool small_mul(stackvec &vec, + limb y) noexcept { limb carry = 0; for (limb_t index = 0; index != vec.len(); ++index) { vec[index] = scalar_mul(vec[index], y, carry); @@ -292,7 +292,7 @@ small_mul(stackvec &vec, limb y) noexcept { // add bigint to bigint starting from index. // used in grade school multiplication template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool large_add_from(stackvec &x, limb_span y, limb_t start) noexcept { // the effective x buffer is from `xstart..x.len()`, so exit early // if we can't get that current range. @@ -324,15 +324,15 @@ large_add_from(stackvec &x, limb_span y, limb_t start) noexcept { // add bigint to bigint. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool large_add_from(stackvec &x, limb_span y) noexcept { return large_add_from(x, y, 0); } // grade-school multiplication algorithm template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool -long_mul(stackvec &x, limb_span y) noexcept { +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool long_mul(stackvec &x, + limb_span y) noexcept { limb_span xs = limb_span(x.data, x.len()); stackvec z(xs); limb_span zs = limb_span(z.data, z.len()); @@ -360,8 +360,8 @@ long_mul(stackvec &x, limb_span y) noexcept { // grade-school multiplication algorithm template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool -large_mul(stackvec &x, limb_span y) noexcept { +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool large_mul(stackvec &x, + limb_span y) noexcept { if (y.len() == 1) { FASTFLOAT_TRY(small_mul(x, y[0])); } else { diff --git a/include/fast_float/decimal_to_binary.h b/include/fast_float/decimal_to_binary.h index 661f8dd..da00f15 100644 --- a/include/fast_float/decimal_to_binary.h +++ b/include/fast_float/decimal_to_binary.h @@ -18,7 +18,7 @@ namespace fast_float { // bits. // template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 value128 +fastfloat_inline FASTFLOAT_CONSTEXPR20 value128 compute_product_approximation(am_pow_t q, am_mant_t w) noexcept { am_pow_t const index = 2 * (q - powers::smallest_power_of_five); // For small values of q, e.g., q in [0,27], the answer is always exact @@ -64,7 +64,7 @@ namespace detail { * where * p = log(5**-q)/log(2) = -q * log(5)/log(2) */ -constexpr fastfloat_really_inline am_pow_t power(am_pow_t const q) noexcept { +constexpr fastfloat_inline am_pow_t power(am_pow_t const q) noexcept { return (((152170 + 65536) * q) >> 16) + 63; } } // namespace detail @@ -72,7 +72,7 @@ constexpr fastfloat_really_inline am_pow_t power(am_pow_t const q) noexcept { // create an adjusted mantissa, biased by the invalid power2 // for significant digits already multiplied by 10 ** q. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 adjusted_mantissa +fastfloat_inline FASTFLOAT_CONSTEXPR14 adjusted_mantissa compute_error_scaled(am_pow_t q, am_mant_t w, limb_t lz) noexcept { auto const hilz = static_cast((w >> 63) ^ 1); adjusted_mantissa answer; @@ -86,7 +86,7 @@ compute_error_scaled(am_pow_t q, am_mant_t w, limb_t lz) noexcept { // w * 10 ** q, without rounding the representation up. // the power2 in the exponent will be adjusted by invalid_am_bias. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa +fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa compute_error(am_pow_t q, am_mant_t w) noexcept { auto const lz = leading_zeroes(w); w <<= lz; @@ -101,17 +101,17 @@ compute_error(am_pow_t q, am_mant_t w) noexcept { // cases, we return an adjusted_mantissa with a negative power of 2: the caller // should recompute in such cases. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa +fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa compute_float(am_pow_t q, am_mant_t w) noexcept { adjusted_mantissa answer; if ((w == 0) || (q < binary::smallest_power_of_ten())) { - // we want to get zero: + // result should be zero answer.power2 = 0; answer.mantissa = 0; return answer; } if (q > binary::largest_power_of_ten()) { - // we want to get infinity: + // result should be infinity answer.power2 = binary::infinite_power(); answer.mantissa = 0; return answer; @@ -152,22 +152,32 @@ compute_float(am_pow_t q, am_mant_t w) noexcept { // Now, we need to round the mantissa correctly. - if (answer.power2 <= 0) { // we have a subnormal or very small value. + if (answer.power2 <= 0) { + // We have a subnormal or very small number. // Here have that answer.power2 <= 0 so -answer.power2 >= 0 - if (-answer.power2 + 1 >= - 64) { // if we have more than 64 bits below the minimum exponent, you - // have a zero for sure. + auto const subnormal_shift = static_cast(-answer.power2 + 1); + if (subnormal_shift >= 64) { + // We have more than 64 bits below the minimum exponent + // result should be zero answer.power2 = 0; answer.mantissa = 0; - // result should be zero return answer; } // We have a subnormal number. We need to shift the mantissa to the right - // next line is safe because -answer.power2 + 1 < 64 - answer.mantissa >>= -answer.power2 + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0 in the 32-bit and - // and 64-bit case (with no more than 19 digits). + answer.mantissa >>= subnormal_shift; + // A subnormal result can also fall exactly between two floats. With at + // most 19 digits this never happens for float and double, but it does for + // std::float16_t (e.g., 2^-25 = 298023223876953125e-25), so we apply the + // same round-to-even test as in the normal case below. + // See script/format_parameters.py. + if (binary::subnormal_ties_possible() && (product.low <= 1) && + (q >= binary::min_exponent_round_to_even()) && + (q <= binary::max_exponent_round_to_even()) && + ((answer.mantissa & 3) == 1)) { + if (((answer.mantissa << subnormal_shift) << shift) == product.high) { + answer.mantissa &= ~am_mant_t(1); // flip it so that we do not round up + } + } answer.mantissa += (answer.mantissa & 1); // round up answer.mantissa >>= 1; // There is a weird scenario where we don't have a subnormal but just. @@ -184,8 +194,8 @@ compute_float(am_pow_t q, am_mant_t w) noexcept { return answer; } - // usually, we round *up*, but if we fall right in between and and we have an - // even basis, we need to round down + // Usually, we round *up*, but if we fall right in between and we have an even + // basis, we need to round down. // We are only concerned with the cases where 5**q fits in single 64-bit word. if ((product.low <= 1) && (q >= binary::min_exponent_round_to_even()) && (q <= binary::max_exponent_round_to_even()) && @@ -203,18 +213,30 @@ compute_float(am_pow_t q, am_mant_t w) noexcept { // Normal rounding answer.mantissa += (answer.mantissa & 1); // round up answer.mantissa >>= 1; - if (answer.mantissa >= (am_mant_t(2) << binary::mantissa_explicit_bits())) { + // Both fix-ups below are rare. They are marked unlikely so that clang keeps + // them as branches instead of folding them into conditional moves that + // every conversion pays for. +#ifdef __clang__ +#pragma clang diagnostic push +#if (!defined(__APPLE_CC__) && __clang_major__ >= 10) || (__clang_major__ >= 13) +#pragma clang diagnostic ignored "-Wc++20-extensions" +#endif +#endif + if fastfloat_unlikely (answer.mantissa >= + (am_mant_t(2) << binary::mantissa_explicit_bits())) { answer.mantissa = (am_mant_t(1) << binary::mantissa_explicit_bits()); - ++answer.power2; // undo previous line addition + ++answer.power2; // undo previous addition } - // Check if we have infinity after computation answer.mantissa &= ~(am_mant_t(1) << binary::mantissa_explicit_bits()); - if (answer.power2 >= binary::infinite_power()) { // infinity + if fastfloat_unlikely (answer.power2 >= binary::infinite_power()) { + // result should be infinity answer.power2 = binary::infinite_power(); answer.mantissa = 0; } - +#ifdef __clang__ +#pragma clang diagnostic pop +#endif return answer; } diff --git a/include/fast_float/digit_comparison.h b/include/fast_float/digit_comparison.h index de7bf00..c03730a 100644 --- a/include/fast_float/digit_comparison.h +++ b/include/fast_float/digit_comparison.h @@ -37,7 +37,7 @@ constexpr static uint64_t powers_of_ten_uint64[] = {1UL, // this algorithm is not even close to optimized, but it has no practical // effect on performance: in order to have a faster algorithm, we'd need // to slow down performance for faster algorithms, and this is still fast. -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 am_pow_t +fastfloat_inline FASTFLOAT_CONSTEXPR14 am_pow_t scientific_exponent(am_mant_t mantissa, am_pow_t exponent) noexcept { while (mantissa >= 10000) { mantissa /= 10000; @@ -56,7 +56,7 @@ scientific_exponent(am_mant_t mantissa, am_pow_t exponent) noexcept { // this converts a native floating-point number to an extended-precision float. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa +fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa to_extended(T const value) noexcept { using equiv_uint = equiv_uint_t; constexpr equiv_uint exponent_mask = binary_format::exponent_mask(); @@ -88,7 +88,7 @@ to_extended(T const value) noexcept { // we are given a native float that represents b, so we need to adjust it // halfway between b and b+u. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa +fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa to_extended_halfway(T const value) noexcept { adjusted_mantissa am = to_extended(value); am.mantissa <<= 1; @@ -99,8 +99,8 @@ to_extended_halfway(T const value) noexcept { // round an extended-precision float to the nearest machine float. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am, - callback cb) noexcept { +fastfloat_inline FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am, + callback cb) noexcept { constexpr am_pow_t mantissa_shift = 64 - binary_format::mantissa_explicit_bits() - 1; if (-am.power2 >= mantissa_shift) { @@ -134,7 +134,7 @@ fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am, } template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void +fastfloat_inline FASTFLOAT_CONSTEXPR14 void round_nearest_tie_even(adjusted_mantissa &am, am_pow_t shift, callback cb) noexcept { am_mant_t const mask = (shift == 64) ? std::numeric_limits::max() @@ -156,7 +156,7 @@ round_nearest_tie_even(adjusted_mantissa &am, am_pow_t shift, am.mantissa += am_mant_t(cb(is_odd, is_halfway, is_above)); } -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void +fastfloat_inline FASTFLOAT_CONSTEXPR14 void round_down(adjusted_mantissa &am, am_pow_t shift) noexcept { if (shift == 64) { am.mantissa = 0; @@ -167,7 +167,7 @@ round_down(adjusted_mantissa &am, am_pow_t shift) noexcept { } template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void +fastfloat_inline FASTFLOAT_CONSTEXPR20 void skip_zeros(UC const *&first, UC const *last) noexcept { if (!is_constant_evaluated()) { while (std::distance(first, last) >= int_cmp_len()) { @@ -190,7 +190,7 @@ skip_zeros(UC const *&first, UC const *last) noexcept { // determine if any non-zero digits were truncated. // all characters must be valid digits. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool is_truncated(UC const *first, UC const *last) noexcept { // do 8-bit optimizations, can just compare to 8 literal 0s. if (!is_constant_evaluated()) { @@ -213,14 +213,14 @@ is_truncated(UC const *first, UC const *last) noexcept { } template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool is_truncated(span s) noexcept { return is_truncated(s.ptr, s.ptr + s.len()); } #if defined(FASTFLOAT_64BIT_LIMB) && defined(FASTFLOAT_X86_SIMD) && \ FASTFLOAT_X86_SIMD >= 31 -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void +fastfloat_inline FASTFLOAT_CONSTEXPR20 void parse_sixteen_digits(char const *&p, limb &value, am_digits &counter, am_digits &count) noexcept { value = parse_16_digits(p); @@ -231,7 +231,7 @@ parse_sixteen_digits(char const *&p, limb &value, am_digits &counter, #endif template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void +fastfloat_inline FASTFLOAT_CONSTEXPR20 void parse_eight_digits(UC const *&p, limb &value, am_digits &counter, am_digits &count) noexcept { value = value * 100000000 + parse_8_digits(p); @@ -241,7 +241,7 @@ parse_eight_digits(UC const *&p, limb &value, am_digits &counter, } template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void +fastfloat_inline FASTFLOAT_CONSTEXPR14 void parse_four_digits(UC const *&p, limb &value, am_digits &counter, am_digits &count) noexcept { value = value * 10000 + parse_4_digits(read_chars_to_unsigned(p)); @@ -251,7 +251,7 @@ parse_four_digits(UC const *&p, limb &value, am_digits &counter, } template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void +fastfloat_inline FASTFLOAT_CONSTEXPR14 void parse_one_digit(UC const *&p, limb &value, am_digits &counter, am_digits &count) noexcept { value = value * 10 + limb(*p - UC('0')); @@ -260,13 +260,13 @@ parse_one_digit(UC const *&p, limb &value, am_digits &counter, ++count; } -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void -add_native(bigint &big, limb power, limb value) noexcept { +fastfloat_inline FASTFLOAT_CONSTEXPR20 void add_native(bigint &big, limb power, + limb value) noexcept { big.mul(power); big.add(value); } -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void +fastfloat_inline FASTFLOAT_CONSTEXPR20 void round_up_bigint(bigint &big, am_digits &count) noexcept { // need to round-up the digits, but need to avoid rounding // ....9999 to ...10000, which could cause a false halfway point. @@ -388,9 +388,8 @@ parse_mantissa(bigint &result, const parsed_number_string_t &num) noexcept { } template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa -positive_digit_comp(bigint &bigmant, adjusted_mantissa am, - am_pow_t const exponent) noexcept { +fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa positive_digit_comp( + bigint &bigmant, adjusted_mantissa am, am_pow_t const exponent) noexcept { FASTFLOAT_ASSERT(bigmant.pow10(exponent)); bool truncated; am.mantissa = bigmant.hi64(truncated); @@ -417,7 +416,7 @@ positive_digit_comp(bigint &bigmant, adjusted_mantissa am, // we then need to scale by `2^(f- e)`, and then the two significant digits // are of the same magnitude. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa +fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa negative_digit_comp(bigint &real_digits, adjusted_mantissa am, am_pow_t const real_exp) noexcept { // get the value of `b`, rounded down, and get a bigint representation of b+h diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index 32fbfc4..0c44d88 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -254,13 +254,13 @@ static_assert(FASTFLOAT_X86_SIMD == 20 || FASTFLOAT_X86_SIMD == 42 || #endif #ifdef FASTFLOAT_VISUAL_STUDIO -#define fastfloat_really_inline __forceinline +#define fastfloat_inline __forceinline #define fastfloat_noinline __declspec(noinline) #elif defined(__GNUC__) || defined(__clang__) -#define fastfloat_really_inline inline __attribute__((always_inline)) +#define fastfloat_inline inline __attribute__((always_inline)) #define fastfloat_noinline __attribute__((noinline)) #else -#define fastfloat_really_inline inline +#define fastfloat_inline inline #define fastfloat_noinline #endif @@ -325,7 +325,7 @@ namespace fast_float { using std::bit_cast; #else template -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 To bit_cast(const From &from) { +fastfloat_inline FASTFLOAT_CONSTEXPR14 To bit_cast(const From &from) { // Implementation of std::bit_cast for pre-C++20. auto to = To(); // The cast suppresses a bogus -Wclass-memaccess on GCC. @@ -337,7 +337,7 @@ fastfloat_really_inline FASTFLOAT_CONSTEXPR14 To bit_cast(const From &from) { #if FASTFLOAT_HAS_IS_CONSTANT_EVALUATED using std::is_constant_evaluated; #else -fastfloat_really_inline constexpr bool is_constant_evaluated() noexcept { +fastfloat_inline constexpr bool is_constant_evaluated() noexcept { return false; } #endif @@ -384,135 +384,23 @@ struct is_supported_char_type }; #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN - +// Compares two ASCII strings in a case insensitive manner. The expected +// string is lowercase ASCII, so OR-ing 0x20 into each actual character maps +// 'A'..'Z' onto 'a'..'z' and leaves everything else mismatched. The lengths +// used are tiny constants (3 and 5), so the compiler unrolls this loop; keep +// it small so that the (rarely executed) inf/nan path does not bloat the hot +// parser it is inlined into. template -inline FASTFLOAT_CONSTEXPR20 bool -fastfloat_strncasecmp3(UC const *actual_mixedcase, - UC const *expected_lowercase) { - if (is_constant_evaluated()) { - for (uint_fast8_t i = 0; i != 3; ++i) { - if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { - return false; - } - } - return true; - } else { - uint64_t val1{0}, val2{0}; - uint64_t mask{0}; - if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 1) { - mask = 0x2020202020202020; - } else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 2) { - mask = 0x0020002000200020; - } else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 4) { - mask = 0x0000002000000020; - } - if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 1 || sizeof(UC) == 2) { - std::memcpy(&val1, actual_mixedcase, 3 * sizeof(UC)); - std::memcpy(&val2, expected_lowercase, 3 * sizeof(UC)); - val1 |= mask; - val2 |= mask; - return val1 == val2; - } else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 4) { - std::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC)); - std::memcpy(&val2, expected_lowercase, 2 * sizeof(UC)); - val1 |= mask; - if (val1 != val2) { - return false; - } - return (actual_mixedcase[2] | 32) == (expected_lowercase[2]); - } else { - return false; - } - } -} - -template -inline FASTFLOAT_CONSTEXPR20 bool -fastfloat_strncasecmp5(UC const *actual_mixedcase, - UC const *expected_lowercase) noexcept { - if (is_constant_evaluated()) { - for (uint_fast8_t i = 0; i != 5; ++i) { - if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { - return false; - } - } - return true; - } else { - uint64_t val1{0}, val2{0}; - if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 1) { - constexpr uint64_t mask = 0x2020202020202020; - std::memcpy(&val1, actual_mixedcase, 5 * sizeof(UC)); - std::memcpy(&val2, expected_lowercase, 5 * sizeof(UC)); - val1 |= mask; - val2 |= mask; - return val1 == val2; - } else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 2) { - constexpr uint64_t mask = 0x0020002000200020; - std::memcpy(&val1, actual_mixedcase, 4 * sizeof(UC)); - std::memcpy(&val2, expected_lowercase, 4 * sizeof(UC)); - val1 |= mask; - if (val1 != val2) { - return false; - } - return (actual_mixedcase[4] | 32) == (expected_lowercase[4]); - } else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 4) { - constexpr uint64_t mask = 0x0000002000000020; - std::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC)); - std::memcpy(&val2, expected_lowercase, 2 * sizeof(UC)); - val1 |= mask; - if (val1 != val2) { - return false; - } - std::memcpy(&val1, actual_mixedcase + 2, 2 * sizeof(UC)); - std::memcpy(&val2, expected_lowercase + 2, 2 * sizeof(UC)); - val1 |= mask; - if (val1 != val2) { - return false; - } - return (actual_mixedcase[4] | 32) == (expected_lowercase[4]); - } else { - return false; - } - } -} - -// Compares two ASCII strings in a case insensitive manner. -template -inline FASTFLOAT_CONSTEXPR20 bool +inline FASTFLOAT_CONSTEXPR14 bool fastfloat_strncasecmp(UC const *actual_mixedcase, UC const *expected_lowercase, uint_fast8_t const length) noexcept { - if (is_constant_evaluated()) { - for (uint_fast8_t i = 0; i != length; ++i) { - if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { - return false; - } - } - return true; - } else { - uint64_t val1{0}, val2{0}; - uint64_t mask{0}; - if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 1) { - mask = 0x2020202020202020; - } else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 2) { - mask = 0x0020002000200020; - } else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 4) { - mask = 0x0000002000000020; - } - uint_fast8_t sz = 8 / (sizeof(UC)); - for (uint_fast8_t i = 0; i < length; i += sz) { - sz = sz < (length - i) ? sz : length - i; - std::memcpy(&val1, actual_mixedcase + i, sz * sizeof(UC)); - std::memcpy(&val2, expected_lowercase + i, sz * sizeof(UC)); - val1 |= mask; - val2 |= mask; - if (val1 != val2) { - return false; - } + for (uint_fast8_t i = 0; i != length; ++i) { + if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { + return false; } } return true; } - #endif #ifndef FLT_EVAL_METHOD @@ -548,7 +436,7 @@ struct value128 { }; /* Helper C++14 constexpr generic implementation of leading_zeroes for 64-bit */ -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 limb_t +fastfloat_inline FASTFLOAT_CONSTEXPR14 limb_t leading_zeroes_generic(uint64_t input_num) noexcept { assert(input_num > 0); FASTFLOAT_ASSUME(input_num > 0); @@ -580,7 +468,7 @@ leading_zeroes_generic(uint64_t input_num) noexcept { } /* result might be undefined when input_num is zero */ -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 limb_t leading_zeroes(uint64_t input_num) noexcept { assert(input_num > 0); FASTFLOAT_ASSUME(input_num > 0); @@ -615,7 +503,7 @@ leading_zeroes(uint64_t input_num) noexcept { } /* Helper C++14 constexpr generic implementation of countr_zero for 32-bit */ -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 limb_t +fastfloat_inline FASTFLOAT_CONSTEXPR14 limb_t countr_zero_generic_32(uint32_t input_num) noexcept { assert(input_num > 0); FASTFLOAT_ASSUME(input_num > 0); @@ -643,7 +531,7 @@ countr_zero_generic_32(uint32_t input_num) noexcept { } /* count trailing zeroes for 32-bit integers */ -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 limb_t countr_zero_32(uint32_t input_num) noexcept { if (is_constant_evaluated()) { return countr_zero_generic_32(input_num); @@ -659,12 +547,12 @@ countr_zero_32(uint32_t input_num) noexcept { #endif } -fastfloat_really_inline constexpr uint64_t emulu_generic(uint32_t x, +fastfloat_inline constexpr uint64_t emulu_generic(uint32_t x, uint32_t y) noexcept { return x * static_cast(y); } -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t umul128_generic(uint64_t ab, uint64_t cd, uint64_t &hi) noexcept { auto ab_shifted = static_cast(ab >> 32); auto cd_shifted = static_cast(cd >> 32); @@ -683,7 +571,7 @@ umul128_generic(uint64_t ab, uint64_t cd, uint64_t &hi) noexcept { } // Compute hi and low parts of 128-bit. -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t umul128(uint64_t ab, uint64_t cd, uint64_t &hi) noexcept { if (is_constant_evaluated()) { return umul128_generic(ab, cd, hi); @@ -702,7 +590,7 @@ umul128(uint64_t ab, uint64_t cd, uint64_t &hi) noexcept { } // Compute 128-bit result. -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 value128 +fastfloat_inline FASTFLOAT_CONSTEXPR20 value128 full_multiplication(uint64_t a, uint64_t b) noexcept { value128 answer; #if defined(FASTFLOAT_64BIT) && defined(__SIZEOF_INT128__) @@ -753,6 +641,8 @@ template struct binary_format : binary_format_lookup_tables { static constexpr am_mant_t max_mantissa_fast_path(am_pow_t const power); static constexpr am_mant_t max_mantissa_fast_path(); // used when fegetround() == FE_TONEAREST + static constexpr bool fast_path_can_overflow(); + static constexpr bool subnormal_ties_possible(); static constexpr am_pow_t largest_power_of_ten(); static constexpr am_pow_t smallest_power_of_ten(); static constexpr T exact_power_of_ten(am_pow_t const power); @@ -784,6 +674,7 @@ template struct binary_format_lookup_tables { 0x20000000000000 / (constant_55555 * constant_55555 * 5), 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5), 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5 * 5 * 5), 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555), 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * 5), 0x20000000000000 / @@ -799,9 +690,7 @@ template struct binary_format_lookup_tables { 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * constant_55555 * 5 * 5), 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * - constant_55555 * 5 * 5 * 5), - 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * - constant_55555 * 5 * 5 * 5 * 5)}; + constant_55555 * 5 * 5 * 5)}; }; #if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE @@ -1033,7 +922,9 @@ binary_format::max_mantissa_fast_path(am_pow_t power) { template <> inline constexpr am_bits_t binary_format::min_exponent_fast_path() { - return 0; + // w / 10^k with w <= 2^11 and k <= 4 rounds correctly even when evaluated + // in float or double first (checked in script/format_parameters.py). + return -4; } template <> @@ -1045,7 +936,9 @@ binary_format::max_exponent_round_to_even() { template <> inline constexpr am_pow_t binary_format::min_exponent_round_to_even() { - return -22; + // -22 covers the normal ties; subnormal ties such as + // 2^-25 = 298023223876953125e-25 need q = -25 and q = -26. + return -26; } template <> @@ -1076,7 +969,8 @@ binary_format::largest_power_of_ten() { template <> inline constexpr am_pow_t binary_format::smallest_power_of_ten() { - return -27; + // (10^19 - 1) * 10^-27 < 2^-25, so any q < -26 rounds to zero. + return -26; } template <> @@ -1173,7 +1067,8 @@ binary_format::max_mantissa_fast_path(am_pow_t power) { template <> inline constexpr am_bits_t binary_format::min_exponent_fast_path() { - return 0; + // Same argument as for std::float16_t (w <= 2^8, k <= 3). + return -3; } template <> @@ -1216,7 +1111,8 @@ binary_format::largest_power_of_ten() { template <> inline constexpr am_pow_t binary_format::smallest_power_of_ten() { - return -60; + // (10^19 - 1) * 10^-60 < 2^-134, so any q < -59 rounds to zero. + return -59; } template <> @@ -1225,6 +1121,25 @@ inline constexpr am_digits binary_format::max_digits() { } #endif // __STDCPP_BFLOAT16_T__ +// Whether Clinger's fast path can overflow: only for std::float16_t, where +// 2^11 * 10^4 > 65504. +template +inline constexpr bool binary_format::fast_path_can_overflow() { + return double(max_mantissa_fast_path()) * + double(exact_power_of_ten(max_exponent_fast_path())) > + double((std::numeric_limits::max)()); +} + +// A subnormal needs w * 10^q < 2^(minimum_exponent() + 1), so q is at most +// (minimum_exponent() + 1) * log10(2), with 1233/4096 < log10(2). Only +// std::float16_t has such q in its round-to-even range. We compare +// 4096 * q with (minimum_exponent() + 1) * 1233 to avoid right-shifting a +// negative value (implementation-defined before C++20). +template +inline constexpr bool binary_format::subnormal_ties_possible() { + return min_exponent_round_to_even() * 4096 <= (minimum_exponent() + 1) * 1233; +} + template <> inline constexpr am_mant_t binary_format::max_mantissa_fast_path(am_pow_t power) { @@ -1342,7 +1257,7 @@ binary_format::hidden_bit_mask() { } template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void to_float( +fastfloat_inline FASTFLOAT_CONSTEXPR20 void to_float( #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN bool const negative, #endif @@ -1502,7 +1417,7 @@ template constexpr uint64_t int_luts::min_safe_u64[]; #endif template -fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) noexcept { +fastfloat_inline constexpr uint8_t ch_to_digit(UC c) noexcept { // wchar_t and char can be signed, so we need to be careful. using UnsignedUC = typename std::make_unsigned::type; return int_luts<>::chdigit[static_cast( @@ -1511,13 +1426,13 @@ fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) noexcept { -((static_cast(c) & ~0xFFull) == 0)))]; } -fastfloat_really_inline constexpr limb_t max_digits_u64(base_t base) noexcept { +fastfloat_inline constexpr limb_t max_digits_u64(base_t base) noexcept { return int_luts<>::maxdigits_u64[base - 2]; } // If a u64 is exactly max_digits_u64() in length, this is // the value below which it has definitely overflowed. -fastfloat_really_inline constexpr uint64_t min_safe_u64(base_t base) noexcept { +fastfloat_inline constexpr uint64_t min_safe_u64(base_t base) noexcept { return int_luts<>::min_safe_u64[base - 2]; } @@ -1594,21 +1509,6 @@ constexpr chars_format operator^(chars_format lhs, chars_format rhs) noexcept { static_cast(rhs)); } -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 chars_format & -operator&=(chars_format &lhs, chars_format rhs) noexcept { - return lhs = (lhs & rhs); -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 chars_format & -operator|=(chars_format &lhs, chars_format rhs) noexcept { - return lhs = (lhs | rhs); -} - -fastfloat_really_inline FASTFLOAT_CONSTEXPR14 chars_format & -operator^=(chars_format &lhs, chars_format rhs) noexcept { - return lhs = (lhs ^ rhs); -} - } // namespace fast_float #endif diff --git a/include/fast_float/parse_number.h b/include/fast_float/parse_number.h index 1b98011..f32c9d0 100644 --- a/include/fast_float/parse_number.h +++ b/include/fast_float/parse_number.h @@ -38,7 +38,7 @@ from_chars_result_t } if (last - first >= 3) { - if (fastfloat_strncasecmp3(first, str_const_nan())) { + if (fastfloat_strncasecmp(first, str_const_nan(), 3)) { answer.ptr = (first += 3); value = minusSign ? -std::numeric_limits::quiet_NaN() : std::numeric_limits::quiet_NaN(); @@ -58,9 +58,9 @@ from_chars_result_t answer.ec = std::errc(); return answer; } - if (fastfloat_strncasecmp3(first, str_const_inf())) { + if (fastfloat_strncasecmp(first, str_const_inf(), 3)) { if ((last - first >= 8) && - fastfloat_strncasecmp5(first + 3, str_const_inf() + 3)) { + fastfloat_strncasecmp(first + 3, str_const_inf() + 3, 5)) { answer.ptr = first + 8; } else { answer.ptr = first + 3; @@ -82,7 +82,7 @@ from_chars_result_t * It is the default on most system. This function is meant to be inexpensive. * Credit : @mwalcott3 */ -fastfloat_really_inline bool rounds_to_nearest() noexcept { +fastfloat_inline bool rounds_to_nearest() noexcept { // https://lemire.me/blog/2020/06/26/gcc-not-nearest/ #if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) return false; @@ -147,7 +147,7 @@ fastfloat_really_inline bool rounds_to_nearest() noexcept { template struct from_chars_caller { template - FASTFLOAT_CONSTEXPR20 static from_chars_result_t + fastfloat_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t call(UC const *first, UC const *last, T &value, parse_options_t const options) noexcept { return from_chars_advanced(first, last, value, options); @@ -157,7 +157,7 @@ template struct from_chars_caller { #ifdef __STDCPP_FLOAT32_T__ template <> struct from_chars_caller { template - FASTFLOAT_CONSTEXPR20 static from_chars_result_t + fastfloat_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t call(UC const *first, UC const *last, std::float32_t &value, parse_options_t const options) noexcept { // if std::float32_t is defined, and we are in C++23 mode; macro set for @@ -171,7 +171,7 @@ template <> struct from_chars_caller { #ifdef __STDCPP_FLOAT64_T__ template <> struct from_chars_caller { template - FASTFLOAT_CONSTEXPR20 static from_chars_result_t + fastfloat_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t call(UC const *first, UC const *last, std::float64_t &value, parse_options_t const options) noexcept { // if std::float64_t is defined, and we are in C++23 mode; macro set for @@ -182,16 +182,31 @@ template <> struct from_chars_caller { }; #endif +// Parser instantiated for a format fixed at compile time, so that every test +// on the format folds away inside it. GCC gets the same effect by inlining the +// whole parser into each caller, where the format is a constant; clang keeps +// it out of line and would otherwise re-test each format flag per conversion. +template +fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_fixed_format(UC const *first, UC const *last, T &value) noexcept { + return from_chars_caller::call(first, last, value, + parse_options_t(Fmt)); +} + template -FASTFLOAT_CONSTEXPR20 from_chars_result_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t from_chars(UC const *first, UC const *last, T &value, chars_format const fmt /*= chars_format::general*/) noexcept { + if (fmt == chars_format::general) { + return from_chars_fixed_format(first, last, + value); + } return from_chars_caller::call(first, last, value, parse_options_t(fmt)); } template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool +fastfloat_inline FASTFLOAT_CONSTEXPR20 bool clinger_fast_path_impl(am_mant_t const mantissa, am_pow_t const exponent, #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN bool const is_negative, @@ -225,21 +240,25 @@ clinger_fast_path_impl(am_mant_t const mantissa, am_pow_t const exponent, ) { // We have that fegetround() == FE_TONEAREST. // Next is Clinger's fast path. - if (mantissa <= binary_format::max_mantissa_fast_path()) { - value = static_cast(mantissa); - if (exponent < 0) { - value = value / binary_format::exact_power_of_ten(-exponent); - } else { - value = value * binary_format::exact_power_of_ten(exponent); + value = static_cast(mantissa); + if (exponent < 0) { + value = value / binary_format::exact_power_of_ten(-exponent); + } else { + value = value * binary_format::exact_power_of_ten(exponent); + // Only std::float16_t can overflow here (e.g., "656e2"); let the + // slow path report result_out_of_range. + if (binary_format::fast_path_can_overflow() && + value > (std::numeric_limits::max)()) { + return false; } -#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN - if (is_negative) { - value = -value; - } -#endif - return true; -#ifndef FASTFLOAT_ONLY_ROUNDS_TO_NEAREST_SUPPORTED } +#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN + if (is_negative) { + value = -value; + } +#endif + return true; +#ifndef FASTFLOAT_ONLY_ROUNDS_TO_NEAREST_SUPPORTED } else { // We do not have that fegetround() == FE_TONEAREST. // Next is a modified Clinger's fast path, inspired by Jakub Jelínek's @@ -265,8 +284,8 @@ clinger_fast_path_impl(am_mant_t const mantissa, am_pow_t const exponent, } #endif return true; -#endif } +#endif } } return false; @@ -278,7 +297,7 @@ clinger_fast_path_impl(am_mant_t const mantissa, am_pow_t const exponent, * parsing options or other parsing custom function implemented by user. */ template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t from_chars_advanced(parsed_number_string_t const &pns, T &value) noexcept { static_assert(is_supported_float_type::value, "this type of floating-point type isn't supported"); @@ -341,14 +360,50 @@ parse_number_slow_path(UC const *first, UC const *last, T &value, ) noexcept { parsed_number_string_t const pns = #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN - bjf ? parse_number_string(first, last, options, true) : + bjf ? parse_number_string_impl(first, last, options, + true) + : #endif - parse_number_string(first, last, options, true); + parse_number_string_impl(first, last, options, + true); + return from_chars_advanced(pns, value); } +#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN +// Cold: the whole chars_format::javascript conversion. Kept out of line and out +// of from_chars_float_advanced so the common frame holds neither the javascript +// parser body nor a call that would force the parsed number string onto the +// stack. Mirrors the main path, only with the javascript parser. template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t +fastfloat_noinline FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_float_javascript(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + if (uint64_t(fmt & chars_format::skip_white_space)) { + while ((first != last) && fast_float::is_space(*first)) { + first++; + } + } + from_chars_result_t answer; + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + if (first == last) { + return answer; + } + parsed_number_string_t pns = + parse_number_string_javascript(first, last, options, true); + if (pns.invalid) { + if (uint64_t(fmt & chars_format::no_infnan)) { + return answer; + } + return detail::parse_infnan(first, last, value, fmt); + } + return from_chars_advanced(pns, value); +} +#endif + +template +fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t from_chars_float_advanced(UC const *first, UC const *last, T &value, parse_options_t const options) noexcept { static_assert(is_supported_float_type::value, @@ -356,6 +411,15 @@ from_chars_float_advanced(UC const *first, UC const *last, T &value, static_assert(is_supported_char_type::value, "only char, wchar_t, char16_t and char32_t are supported"); +#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN + // Leave for the javascript conversion before anything else is live, so this + // compiles to a tail call and the common path keeps its registers. The cold + // function repeats the leading-whitespace and empty-input handling below. + if fastfloat_unlikely (uint64_t(fmt & detail::basic_javascript_fmt)) { + return from_chars_float_javascript(first, last, value, options); + } +#endif + from_chars_result_t answer; #ifdef FASTFLOAT_ISNOT_CHECKED_BOUNDS // We are in parser code with external loop that checks bounds. @@ -391,12 +455,15 @@ from_chars_float_advanced(UC const *first, UC const *last, T &value, // Fast path: parse WITHOUT materializing the integer/fraction spans (read // only by the rare slow paths). Skipping their stores keeps the fat // parsed_number_string_t off the hot path. store_spans is a runtime argument, - // so this reuses the single parse_number_string instantiation. + // so this reuses the single parse_number_string_impl instantiation. parsed_number_string_t const pns = #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN - bjf ? parse_number_string(first, last, options, false) : + bjf ? parse_number_string_impl(first, last, options, + false) + : #endif - parse_number_string(first, last, options, false); + parse_number_string_impl(first, last, options, + false); if (pns.invalid) { #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN if (chars_format_t(options.format & chars_format::no_infnan)) { @@ -458,19 +525,22 @@ from_chars_float_advanced(UC const *first, UC const *last, T &value, #endif ); } -#ifdef __clang__ -#pragma clang diagnostic pop -#endif to_float( #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN pns.negative, #endif am, value); - // Test for over/underflow. - if ((pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) || - am.power2 == binary_format::infinite_power()) { + // Test for over/underflow. Marked unlikely so that clang keeps it as a + // branch instead of folding it into a chain of conditional moves that + // every conversion pays for. + if fastfloat_unlikely ((pns.mantissa != 0 && am.mantissa == 0 && + am.power2 == 0) || + am.power2 == binary_format::infinite_power()) { answer.ec = std::errc::result_out_of_range; } +#ifdef __clang__ +#pragma clang diagnostic pop +#endif return answer; } @@ -493,15 +563,15 @@ FASTFLOAT_CONSTEXPR20 integer_times_pow10(uint64_t const mantissa, int const decimal_exponent) noexcept { T value; - const auto exponent = static_cast(decimal_exponent); - if (clinger_fast_path_impl(mantissa, exponent, + if (clinger_fast_path_impl(mantissa, static_cast(decimal_exponent), #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN false, #endif value)) return value; - adjusted_mantissa am = compute_float>(exponent, mantissa); + adjusted_mantissa am = compute_float>( + static_cast(decimal_exponent), mantissa); to_float( #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN false, @@ -522,16 +592,16 @@ FASTFLOAT_CONSTEXPR20 FASTFLOAT_ASSUME(mantissa >= 0); const auto m = static_cast(mantissa); #endif - const auto exponent = static_cast(decimal_exponent); T value; - if (clinger_fast_path_impl(m, exponent, + if (clinger_fast_path_impl(m, static_cast(decimal_exponent), #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN is_negative, #endif value)) return value; - adjusted_mantissa const am = compute_float>(exponent, m); + adjusted_mantissa const am = compute_float>( + static_cast(decimal_exponent), m); to_float( #ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN @@ -634,7 +704,7 @@ template struct from_chars_advanced_caller { template <> struct from_chars_advanced_caller<1> { template - fastfloat_really_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t + fastfloat_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t call(UC const *first, UC const *last, T &value, parse_options_t const options) noexcept { return from_chars_float_advanced(first, last, value, options); @@ -643,7 +713,7 @@ template <> struct from_chars_advanced_caller<1> { template <> struct from_chars_advanced_caller<2> { template - fastfloat_really_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t + fastfloat_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t call(UC const *first, UC const *last, T &value, parse_options_t const options) noexcept { return from_chars_int_advanced(first, last, value, options); @@ -651,7 +721,7 @@ template <> struct from_chars_advanced_caller<2> { }; template -fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t +fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t from_chars_advanced(UC const *first, UC const *last, T &value, parse_options_t const options) noexcept { return from_chars_advanced_caller< diff --git a/script/format_parameters.py b/script/format_parameters.py new file mode 100644 index 0000000..e981b8f --- /dev/null +++ b/script/format_parameters.py @@ -0,0 +1,416 @@ +# +# Checks the binary_format constants in include/fast_float/float_common.h +# with exact integer arithmetic, for double, float, std::float16_t and +# std::bfloat16_t. Every string is parsed as w * 10^q with w < 10^19, so each +# constant has a condition on (w, q) that it must satisfy. +# +# References: +# Daniel Lemire, Number Parsing at a Gigabyte per Second, +# Software: Practice and Experience 51 (8), 2021 https://arxiv.org/abs/2101.11408 +# Noble Mushtak and Daniel Lemire, Fast Number Parsing Without Fallback, +# Software: Practice and Experience 53 (6), 2023 https://arxiv.org/abs/2212.06644 +# +# Usage: python3 script/format_parameters.py [--types float16,bfloat16] +# [--sweep-width 3] [--jobs N] +# + +import argparse +import multiprocessing +import os +import re +import sys + +HEADER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", + "include", "fast_float", "float_common.h") +MASK64 = (1 << 64) - 1 +MAX_W = 10**19 +CXX_TYPES = {"double": "double", "float": "float", + "float16": "std::float16_t", "bfloat16": "std::bfloat16_t"} + + +class Format: + def __init__(self, name): + self.name = name + cxx = CXX_TYPES[name] + with open(HEADER) as f: + text = f.read() + pattern = r"binary_format<" + re.escape(cxx) + r">::(\w+)\(\)\s*\{(.*?)\n\}" + for fn, body in re.findall(pattern, text, re.S): + returns = re.findall(r"return\s+([^;]+);", body) + # the last return is the FLT_EVAL_METHOD 0/1 branch + if returns and re.fullmatch(r"-?(0x[0-9A-Fa-f]+|\d+)", returns[-1].strip()): + setattr(self, fn, int(returns[-1], 0)) + tables = re.search(r"binary_format_lookup_tables<" + re.escape(cxx) + + r", U>\s*\{(.*?)\n\};", text, re.S).group(1) + powers = re.search(r"powers_of_ten\[\]\s*=\s*\{(.*?)\}", tables, re.S).group(1) + self.powers_of_ten = [float(x) for x in + re.sub(r"(bf16|f16|f)\b", "", powers).split(",") if x.strip()] + mant = re.search(r"max_mantissa\[\]\s*=\s*\{(.*?)\}", tables, re.S).group(1) + mant = re.sub(r"//[^\n]*", "", mant).replace("/", "//").replace("\n", " ") + self.max_mantissa = [eval(x.strip(), {"constant_55555": 5**5}) + for x in mant.split(",") if x.strip()] + self.p = self.mantissa_explicit_bits + self.bias = self.p - self.minimum_exponent + self.denorm_min_exp = 1 - self.bias # smallest subnormal is 2^denorm_min_exp + self.emin = self.minimum_exponent + 1 # leading bit of the smallest normal + self.emax = self.infinite_power - 1 + self.minimum_exponent + + def grid(self): + """Every finite value and every midpoint as (num, exp2, kind) with + kind in value, tie_down (even lower neighbour), tie_up, overflow.""" + out = [] + two_p = 1 << self.p + d = self.denorm_min_exp + for m in range(1, two_p): + out.append((m, d, "value")) + for m in range(two_p): + out.append((2 * m + 1, d - 1, "tie_down" if m % 2 == 0 else "tie_up")) + for e in range(self.emin, self.emax + 1): + for f in range(two_p): + out.append((two_p + f, e - self.p, "value")) + kind = "tie_down" if f % 2 == 0 else "tie_up" + if e == self.emax and f == two_p - 1: + kind = "overflow" + out.append((2 * (two_p + f) + 1, e - self.p - 1, kind)) + return out + + +def ratio(num, exp2): + return (num << exp2, 1) if exp2 >= 0 else (num, 1 << -exp2) + + +def ratio10(w, q): + return (w * 10**q, 1) if q >= 0 else (w, 10**-q) + + +def le(n1, d1, n2, d2): + return n1 * d2 <= n2 * d1 + + +def floor_log2(n, d): + e = n.bit_length() - d.bit_length() + if (d << e if e >= 0 else d) > (n if e >= 0 else n << -e): + e -= 1 + return e + + +def round_ties_even(n, d): + m, r = divmod(n, d) + if 2 * r > d or (2 * r == d and m & 1): + m += 1 + return m + + +def reference_bits(fmt, n, d): + """Correctly rounded bit pattern of n/d >= 0.""" + if n == 0: + return 0 + e = floor_log2(n, d) + s = fmt.denorm_min_exp if e < fmt.emin else e - fmt.p + m = round_ties_even(n, d << s) if s >= 0 else round_ties_even(n << -s, d) + if e < fmt.emin: + return m + if m == 2 << fmt.p: + m >>= 1 + e += 1 + if e > fmt.emax: + return fmt.infinite_power << fmt.p + return ((e - fmt.minimum_exponent) << fmt.p) | (m - (1 << fmt.p)) + + +def ceil_log2(x): + z = 0 + while (1 << z) < x: + z += 1 + return z + + +def table_entry(q): + """T[q] as stored in fast_table.h (see table_generation.py).""" + if q < 0: + power5 = 5**-q + z = ceil_log2(power5) + b = z + 127 if q >= -27 else 2 * z + 128 + c = 2**b // power5 + 1 + while c >= 1 << 128: + c //= 2 + return c + power5 = 5**q + while power5 < 1 << 127: + power5 *= 2 + while power5 >= 1 << 128: + power5 //= 2 + return power5 + + +TABLE = {q: table_entry(q) for q in range(-342, 309)} + + +def compute_product_approximation(q, w, bit_precision): + t = TABLE[q] + first = w * (t >> 64) + high, low = first >> 64, first & MASK64 + if (high & (MASK64 >> bit_precision)) == MASK64 >> bit_precision: + second_high = (w * (t & MASK64)) >> 64 + low = (low + second_high) & MASK64 + if second_high > low: + high += 1 + return high, low + + +def compute_float(fmt, q, w, subnormal_tie_test=True): + """Model of compute_float in decimal_to_binary.h. Returns + (mantissa, power2, truncated mantissa, k) with truncated == floor(w 10^q 2^k).""" + if w == 0 or q < fmt.smallest_power_of_ten: + return 0, 0, None, None + if q > fmt.largest_power_of_ten: + return 0, fmt.infinite_power, None, None + lz = 64 - w.bit_length() + w <<= lz + high, low = compute_product_approximation(q, w, fmt.p + 3) + upperbit = high >> 63 + shift = upperbit + 64 - fmt.p - 3 + mantissa = high >> shift + power2 = ((152170 + 65536) * q >> 16) + 63 + upperbit - lz - fmt.minimum_exponent + tie_q = fmt.min_exponent_round_to_even <= q <= fmt.max_exponent_round_to_even + if power2 <= 0: + if -power2 + 1 >= 64: + return 0, 0, None, None + extra = -power2 + 1 + mantissa >>= extra + truncated, k = mantissa, fmt.bias + if (subnormal_tie_test and low <= 1 and tie_q and (mantissa & 3) == 1 + and ((mantissa << extra) << shift) == high): + mantissa &= ~1 + mantissa += mantissa & 1 + mantissa >>= 1 + return mantissa, int(mantissa >= 1 << fmt.p), truncated, k + truncated, k = mantissa, 1 + fmt.bias - power2 + if low <= 1 and tie_q and (mantissa & 3) == 1 and (mantissa << shift) == high: + mantissa &= ~1 + mantissa += mantissa & 1 + mantissa >>= 1 + if mantissa >= 2 << fmt.p: + mantissa = 1 << fmt.p + power2 += 1 + mantissa &= ~(1 << fmt.p) + if power2 >= fmt.infinite_power: + return 0, fmt.infinite_power, truncated, k + return mantissa, power2, truncated, k + + +def parsed_bits(fmt, w, q, subnormal_tie_test=True): + """from_chars on the digits w * 10^q. Clinger's fast path is one correctly + rounded operation on exact operands, so it equals the reference.""" + if fmt.min_exponent_fast_path <= q <= fmt.max_exponent_fast_path and w <= 2 << fmt.p: + return reference_bits(fmt, *ratio10(w, q)), None, None + mantissa, power2, truncated, k = compute_float(fmt, q, w, subnormal_tie_test) + return mantissa | (power2 << fmt.p), truncated, k + + +failures = 0 + + +def check(ok, text): + global failures + print((" ok " if ok else " FAIL ") + text) + failures += not ok + + +def note(text): + print(" note " + text) + + +def continued_fraction(numer, denom): + cf = [] + while denom: + quot, rem = divmod(numer, denom) + cf.append(quot) + numer, denom = denom, rem + return cf + + +def convergents(cf): + p2, q2, p1, q1 = 0, 1, 1, 0 + for a in cf: + p1, p2, q1, q2 = a * p1 + p2, p1, a * q1 + q2, q1 + yield p1, q1 + + +def check_constants(fmt): + half_denorm = ratio(1, fmt.denorm_min_exp - 1) + q = fmt.smallest_power_of_ten + check(le(*ratio10(MAX_W - 1, q - 1), *half_denorm), + "smallest_power_of_ten=%d: q < %d rounds to zero" % (q, q)) + check(not le(*ratio10(MAX_W - 1, q), *half_denorm), + "smallest_power_of_ten is tight") + threshold = ratio((4 << fmt.p) - 1, fmt.emax - fmt.p - 1) # max + ulp/2 + q = fmt.largest_power_of_ten + check(le(*threshold, *ratio10(1, q + 1)), + "largest_power_of_ten=%d: q > %d overflows" % (q, q)) + check(not le(*threshold, *ratio10(1, q)), "largest_power_of_ten is tight") + + two_p1 = 2 << fmt.p + q = fmt.max_exponent_fast_path + check(5**q <= two_p1 < 5 ** (q + 1), + "max_exponent_fast_path=%d: 10^q exact in T iff q <= %d" % (q, q)) + check(fmt.powers_of_ten[: q + 1] == [10.0**i for i in range(q + 1)], "powers_of_ten") + check(fmt.max_mantissa[: q + 1] == [two_p1 // 5**i for i in range(q + 1)], + "max_mantissa[q] == floor(2^%d / 5^q)" % (fmt.p + 1)) + overflow = not le(two_p1 * 10**q, 1, *ratio(two_p1 - 1, fmt.emax - fmt.p)) + note("fast path product 2^%d * 10^%d %s the largest finite value" + % (fmt.p + 1, q, "exceeds" if overflow else "is below")) + k = -fmt.min_exponent_fast_path + check(0 <= k <= fmt.max_exponent_fast_path, "min_exponent_fast_path=%d" % -k) + if fmt.p <= 10: + # double rounding: w / 10^k rounded to `wide` bits, then to T + for wide in (24, 53, 64): + bad = 0 + for kk in range(1, fmt.max_exponent_fast_path + 1): + for w in range(1, two_p1 + 1): + e = floor_log2(w, 10**kk) + s = e - wide + 1 + m = round_ties_even(w, 10**kk << s) if s >= 0 else round_ties_even(w << -s, 10**kk) + if reference_bits(fmt, *ratio(m, s)) != reference_bits(fmt, w, 10**kk): + bad += 1 + check(bad == 0, "w / 10^k, k <= %d, via %d-bit intermediate never double-rounds" + % (fmt.max_exponent_fast_path, wide)) + + +def check_ties(fmt): + """A tie is m * 2^e with m odd. It is w * 10^q, w < 10^19, iff q <= e and + w = m 5^-q 2^(e-q) (q <= 0) or 5^q | m (q > 0). Ties whose lower neighbour + is even (m == 1 mod 4) must fall in the round-to-even range.""" + p = fmt.p + lo, hi = fmt.min_exponent_round_to_even, fmt.max_exponent_round_to_even + q = 0 + while ((1 << (p + 1)) + 1) * 5 ** (q + 1) < MAX_W: + q += 1 + normal_lo = -q + q = 0 + while 5 ** (q + 1) < 1 << (p + 2): + q += 1 + while True: + c = 1 + while 5**q * c <= 1 << (p + 1): + c += 4 + if 5**q * c < 1 << (p + 2) and q <= fmt.emax - p - 1: + break + q -= 1 + normal_hi = q + e = fmt.denorm_min_exp - 1 + sub = [] + q = e + while 5**-q << (e - q) < MAX_W: + sub.append(q) + q -= 1 + check(lo <= normal_lo and hi >= normal_hi, + "round_to_even range [%d, %d] covers normal ties [%d, %d]" % (lo, hi, normal_lo, normal_hi)) + if sub: + check(lo <= min(sub), "and the subnormal ties at q in [%d, %d]" % (min(sub), max(sub))) + else: + note("no subnormal tie has fewer than 20 digits") + # subnormal_ties_possible() in float_common.h: an upper bound on the + # largest q that can give a subnormal, compared with the tie range + q_sub = ((fmt.minimum_exponent + 1) * 1233) >> 12 + check(10**q_sub < 2**fmt.emin < 10 ** (q_sub + 2), "q <= %d bounds the subnormal exponents" % q_sub) + check((lo <= q_sub) == bool(sub), "subnormal_ties_possible() == %s" % bool(sub)) + longest = len(str(((1 << (p + 2)) - 1) * 5**fmt.bias)) + check(fmt.max_digits >= longest, "max_digits=%d >= %d, the longest tie" % (fmt.max_digits, longest)) + + +def check_table(fmt): + """Facts behind the tie test (product.low <= 1 and dropped bits zero).""" + q_lo, q_hi = max(fmt.smallest_power_of_ten, -27), -1 + check(fmt.min_exponent_round_to_even >= -27, "tie test stays where 5^-q < 2^64") + check(all(TABLE[q] & MASK64 for q in range(q_lo, q_hi + 1)), + "T[q] low word nonzero for %d <= q <= %d" % (q_lo, q_hi)) + fake = False + for q in range(q_lo, q_hi + 1): + t_hi = TABLE[q] >> 64 + v2 = (t_hi & -t_hi).bit_length() - 1 + z = ceil_log2(5**-q) + for shift in (61 - fmt.p, 62 - fmt.p): + # skipped second product: w * t_hi == 0 or 1 mod 2^(shift+64) + if shift + 64 - v2 <= 63: + fake = True + if v2 == 0 and (1 << 63) <= pow(t_hi, -1, 1 << (shift + 64)) < 1 << 64: + fake = True + # computed second product: a non-tie within 2^-62 of a rational + # with denominator 2^(z-1-shift) is impossible + if z - 1 - shift > 62: + fake = True + check(not fake, "no false tie for %d <= q <= %d" % (q_lo, q_hi)) + bad = False + for q in range(fmt.smallest_power_of_ten, fmt.largest_power_of_ten + 1): + for _, w in convergents(continued_fraction(TABLE[q], 2**137)): + if w >= 2**64: + break + if (TABLE[q] * w) % 2**137 > 2**137 - 2**64: + bad = True + check(not bad, "Mushtak-Lemire condition for %d <= q <= %d" + % (fmt.smallest_power_of_ten, fmt.largest_power_of_ten)) + + +def sweep_q(args): + fmt, grid, q, width = args + cands = set() + for num, exp2, _ in grid: + n, d = ratio(num, exp2) + if q >= 0: + d *= 10**q + else: + n *= 10**-q + cands.update(w for w in range(n // d - width, n // d + width + 2) if 1 <= w < MAX_W) + wrong, wrong_without, bad_trunc = [], 0, 0 + for w in cands: + n, d = ratio10(w, q) + want = reference_bits(fmt, n, d) + got, truncated, k = parsed_bits(fmt, w, q) + if got != want: + wrong.append((w, q, got, want)) + if truncated is not None: + exact = (n << k) // d if k >= 0 else n // (d << -k) + bad_trunc += truncated != exact + if parsed_bits(fmt, w, q, False)[0] != want: + wrong_without += 1 + return len(cands), wrong, wrong_without, bad_trunc + + +def sweep(fmt, width, jobs): + """Compare the model with the reference on every w within `width` of + G / 10^q for every value, tie and the overflow threshold G.""" + grid = fmt.grid() + qs = range(fmt.smallest_power_of_ten - 1, fmt.largest_power_of_ten + 2) + tasks = [(fmt, grid, q, width) for q in qs] + with multiprocessing.Pool(jobs) as pool: + results = pool.map(sweep_q, tasks) + wrong = [x for r in results for x in r[1]] + note("swept %d pairs (w, q)" % sum(r[0] for r in results)) + check(sum(r[3] for r in results) == 0, "truncated mantissa is exact") + check(not wrong, "compute_float matches the reference") + for w, q, got, want in sorted(wrong)[:20]: + print(" w=%d q=%d got 0x%04x want 0x%04x" % (w, q, got, want)) + note("without the subnormal tie test: %d wrong" % sum(r[2] for r in results)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--types", default="double,float,float16,bfloat16") + ap.add_argument("--sweep-width", type=int, default=3) + ap.add_argument("--jobs", type=int, default=os.cpu_count() or 1) + args = ap.parse_args() + for name in args.types.split(","): + fmt = Format(name.strip()) + print("== %s" % fmt.name) + check_constants(fmt) + check_table(fmt) + check_ties(fmt) + if fmt.p <= 10 and args.sweep_width > 0: + sweep(fmt, args.sweep_width, args.jobs) + print("%d failure(s)" % failures) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a9926d1..84c0bf0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -95,6 +95,8 @@ endif() if (FASTFLOAT_FIXEDWIDTH_TESTS) fast_float_add_cpp_test(fixedwidthtest) target_compile_features(fixedwidthtest PUBLIC cxx_std_23) + fast_float_add_cpp_test(exhaustive16_midpoint) + target_compile_features(exhaustive16_midpoint PUBLIC cxx_std_23) endif() option(FASTFLOAT_EXHAUSTIVE "Exhaustive tests" OFF) diff --git a/tests/basictest.cpp b/tests/basictest.cpp index 7e12442..dfdd501 100644 --- a/tests/basictest.cpp +++ b/tests/basictest.cpp @@ -1828,6 +1828,20 @@ TEST_CASE("float16.inf") { std::errc::result_out_of_range); verify("3.5028234666e38", std::numeric_limits::infinity(), std::errc::result_out_of_range); + // overflow in the fast path + verify("656e2", std::numeric_limits::infinity(), + std::errc::result_out_of_range); + verify("-656e2", -std::numeric_limits::infinity(), + std::errc::result_out_of_range); + verify("7e4", std::numeric_limits::infinity(), + std::errc::result_out_of_range); + verify("2048e4", std::numeric_limits::infinity(), + std::errc::result_out_of_range); + verify("655e2", 0x1.ffcp+15f16); // 65500 rounds to max + // max + ulp/2 rounds to even + verify("6552e1", std::numeric_limits::infinity(), + std::errc::result_out_of_range); + verify("65519", 0x1.ffcp+15f16); } TEST_CASE("float16.general") { @@ -1848,6 +1862,22 @@ TEST_CASE("float16.general") { verify("-0.000000059604644775390625", -0x1p-24f16); verify("-5.9604644775390625e-8", -0x1p-24f16); + // subnormal ties round to even + verify("2.98023223876953125e-8", 0.0f16, std::errc::result_out_of_range); + verify("0.0000000298023223876953125", 0.0f16, std::errc::result_out_of_range); + verify("-2.98023223876953125e-8", -0.0f16, std::errc::result_out_of_range); + verify("8.94069671630859375e-8", 0x1p-23f16); + verify("1.490116119384765625e-7", 0x1p-23f16); + verify("9.834766387939453125e-7", 0x1p-20f16); + // just above or below a tie + verify("2.980232238769531251e-8", 0x1p-24f16); + verify("2.980232238769531249e-8", 0.0f16, std::errc::result_out_of_range); + verify("1.4901161193847656251e-7", 0x1.8p-23f16); + verify("1.4901161193847656249e-7", 0x1p-23f16); + // more than 19 digits + verify("2.9802322387695312500e-8", 0.0f16, std::errc::result_out_of_range); + verify("1.4901161193847656250000e-7", 0x1p-23f16); + verify("-1e-999", -0.0f16, std::errc::result_out_of_range); verify("6.0975551605224609375", 0x1.864p+2f16); verify_runtime(append_zeros("6.0975551605224609375", 655), 0x1.864p+2f16); diff --git a/tests/exhaustive16_midpoint.cpp b/tests/exhaustive16_midpoint.cpp new file mode 100644 index 0000000..816aeeb --- /dev/null +++ b/tests/exhaustive16_midpoint.cpp @@ -0,0 +1,163 @@ +// For std::float16_t and std::bfloat16_t, parse the exact decimal expansion +// of every finite value and of every midpoint between adjacent values, plus +// strings just above and below each midpoint and midpoints padded past 19 +// digits. The expected bits follow from the bit pattern that generated the +// string, so no C library reference is needed. +#include "fast_float/fast_float.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +// Decimal digit strings, most significant digit first, no leading zeros. +std::string times(std::string const &digits, unsigned factor) { + std::string out; + unsigned carry = 0; + for (size_t i = digits.size(); i-- > 0;) { + unsigned v = static_cast(digits[i] - '0') * factor + carry; + out.push_back(static_cast('0' + v % 10)); + carry = v / 10; + } + while (carry != 0) { + out.push_back(static_cast('0' + carry % 10)); + carry /= 10; + } + return std::string(out.rbegin(), out.rend()); +} + +std::string minus_one(std::string digits) { + size_t i = digits.size(); + while (i-- > 0) { + if (digits[i] != '0') { + digits[i] = static_cast(digits[i] - 1); + break; + } + digits[i] = '9'; + } + return digits; +} + +// num * 2^exp2 written as "e" with an exact expansion. +struct Decimal { + std::string digits; + int exponent; + + std::string str() const { return digits + "e" + std::to_string(exponent); } + + Decimal above() const { return {digits + "1", exponent - 1}; } + + Decimal below() const { return {minus_one(digits) + "9", exponent - 1}; } + + Decimal padded(size_t count) const { + return {digits + std::string(count, '0'), + exponent - static_cast(count)}; + } +}; + +Decimal exact_decimal(uint64_t num, int exp2) { + std::string digits = std::to_string(num); + if (exp2 >= 0) { + for (int i = 0; i < exp2; i++) { + digits = times(digits, 2); + } + return {digits, 0}; + } + for (int i = 0; i < -exp2; i++) { + digits = times(digits, 5); + } + return {digits, exp2}; +} + +template struct Layout; + +template <> struct Layout { + static constexpr char const *name = "float16"; + static constexpr int p = 10; + static constexpr int bias = 25; // value = mantissa * 2^(field - bias) + static constexpr uint16_t infinity = 0x7C00; +}; + +template <> struct Layout { + static constexpr char const *name = "bfloat16"; + static constexpr int p = 7; + static constexpr int bias = 134; + static constexpr uint16_t infinity = 0x7F80; +}; + +struct Failure { + long count = 0; + long checked = 0; +}; + +template +void expect(Failure &f, std::string const &s, uint16_t want) { + T value{}; + auto r = fast_float::from_chars(s.data(), s.data() + s.size(), value); + uint16_t got = 0; + std::memcpy(&got, &value, sizeof(got)); + bool zero_or_inf = want == 0 || want == Layout::infinity; + std::errc want_ec = + zero_or_inf ? std::errc::result_out_of_range : std::errc(); + f.checked++; + if (got != want || r.ec != want_ec || r.ptr != s.data() + s.size()) { + f.count++; + if (f.count <= 20) { + std::printf( + "%s: \"%s\" parsed to 0x%04x (ec %d), expected 0x%04x (ec %d)\n", + Layout::name, s.c_str(), got, static_cast(r.ec), want, + static_cast(want_ec)); + } + } +} + +template Failure sweep() { + Failure f; + constexpr int p = Layout::p; + constexpr uint16_t mantissa_mask = static_cast((1u << p) - 1); + for (uint32_t b = 0; b < Layout::infinity; b++) { + uint16_t bits = static_cast(b); + int field = bits >> p; + uint64_t num = bits & mantissa_mask; + int exp2 = (field == 0 ? 1 : field) - Layout::bias; + if (field != 0) { + num |= uint64_t(1) << p; + } + if (num != 0) { + Decimal d = exact_decimal(num, exp2); + expect(f, d.str(), bits); + expect(f, d.padded(21).str(), bits); + } + // The tie between bits and bits + 1 is (2 num + 1) * 2^(exp2 - 1); the + // last one (bits + 1 == infinity) is the overflow threshold. + Decimal tie = exact_decimal(2 * num + 1, exp2 - 1); + uint16_t lower = bits; + uint16_t upper = static_cast(bits + 1); + uint16_t even = (bits & 1) ? upper : lower; + expect(f, tie.str(), even); + expect(f, tie.padded(21).str(), even); + expect(f, tie.above().str(), upper); + expect(f, tie.below().str(), lower); + } + return f; +} + +} // namespace + +int main() { + Failure f16 = sweep(); + std::printf("float16: %ld strings, %ld failures\n", f16.checked, f16.count); + Failure bf16 = sweep(); + std::printf("bfloat16: %ld strings, %ld failures\n", bf16.checked, + bf16.count); + if (f16.count != 0 || bf16.count != 0) { + return EXIT_FAILURE; + } + std::printf("all ok\n"); + return EXIT_SUCCESS; +}