coffee/fast_float
1
0
Fork 0
mirror of https://github.com/fastfloat/fast_float.git synced 2026-10-29 02:17:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup for code related with merge and add some documentation.

Browse Source 
Merge branch 'main' of https://github.com/fastfloat/fast_float
This commit is contained in:
HedgehogInTheCPP 2026-09-23 02:51:34 +03:00
commit ac96536399
10 changed files with 947 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
include/fast_float/ascii_number.h
View File

@ -19,7 +19,7 @@
namespace fast_float {
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEVAL bool has_simd_opt() noexcept {
fastfloat_inline FASTFLOAT_CONSTEVAL bool has_simd_opt() noexcept {
#ifdef FASTFLOAT_USE_SIMD
return std::is_same<UC, char16_t>::value;
#else
@ -30,7 +30,7 @@ fastfloat_really_inline FASTFLOAT_CONSTEVAL bool has_simd_opt() noexcept {
// Next function can be micro-optimized, but compilers are entirely
// able to optimize it well.
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 bool is_integer(UC c) noexcept {
fastfloat_inline FASTFLOAT_CONSTEXPR14 bool is_integer(UC c) noexcept {
const auto d = c - UC('0');
// UC may be signed.
return d >= 0 && d <= 9;
@ -44,14 +44,14 @@ using std::byteswap;
}
#else
fastfloat_really_inline constexpr uint64_t byteswap(uint64_t val) noexcept {
fastfloat_inline constexpr uint64_t byteswap(uint64_t val) noexcept {
return (val & 0xFF00000000000000) >> 56 | (val & 0x00FF000000000000) >> 40 |
(val & 0x0000FF0000000000) >> 24 | (val & 0x000000FF00000000) >> 8 |
(val & 0x00000000FF000000) << 8 | (val & 0x0000000000FF0000) << 24 |
(val & 0x000000000000FF00) << 40 | (val & 0x00000000000000FF) << 56;
}
fastfloat_really_inline constexpr uint32_t byteswap(uint32_t val) noexcept {
fastfloat_inline constexpr uint32_t byteswap(uint32_t val) noexcept {
return (val >> 24) | ((val >> 8) & 0x0000FF00u) | ((val << 8) & 0x00FF0000u) |
(val << 24);
}
@ -62,7 +62,7 @@ fastfloat_really_inline constexpr uint32_t byteswap(uint32_t val) noexcept {
// Read UCs into an unsigned integer. Truncates UC if not char.
template <typename T, typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 T
fastfloat_inline FASTFLOAT_CONSTEXPR20 T
read_chars_to_unsigned(UC const *chars) noexcept {
if (is_constant_evaluated() || !std::is_same<UC, char>::value) {
T val = 0;
@ -85,7 +85,7 @@ read_chars_to_unsigned(UC const *chars) noexcept {
#if FASTFLOAT_X86_SIMD
fastfloat_really_inline uint64_t simd_read8(__m128i const data) {
fastfloat_inline uint64_t simd_read8(__m128i const data) {
// _mm_packus_epi16 is SSE2, converts 8×u16 → 8×u8
__m128i const packed = _mm_packus_epi16(data, data);
@ -104,7 +104,7 @@ fastfloat_really_inline uint64_t simd_read8(__m128i const data) {
#endif
}
fastfloat_really_inline uint64_t simd_read8(char16_t const *chars) {
fastfloat_inline uint64_t simd_read8(char16_t const *chars) {
FASTFLOAT_SIMD_DISABLE_WARNINGS
// unaligned SIMD instruction -> all fine.
return simd_read8(_mm_loadu_si128(reinterpret_cast<__m128i const *>(chars)));
@ -113,12 +113,12 @@ fastfloat_really_inline uint64_t simd_read8(char16_t const *chars) {
#elif FASTFLOAT_ARM_NEON
fastfloat_really_inline uint64_t simd_read8(uint16x8_t const &data) {
fastfloat_inline uint64_t simd_read8(uint16x8_t const &data) {
uint8x8_t utf8_packed = vmovn_u16(data);
return vget_lane_u64(vreinterpret_u64_u8(utf8_packed), 0);
}
fastfloat_really_inline uint64_t simd_read8(char16_t const *chars) {
fastfloat_inline uint64_t simd_read8(char16_t const *chars) {
FASTFLOAT_SIMD_DISABLE_WARNINGS
return simd_read8(vld1q_u16(reinterpret_cast<uint16_t const *>(chars)));
FASTFLOAT_SIMD_RESTORE_WARNINGS
@ -140,7 +140,7 @@ uint64_t simd_read8(UC const *) {
}
// credit @aqrit
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint32_t
fastfloat_inline FASTFLOAT_CONSTEXPR14 uint32_t
parse_8_digits(uint64_t val) noexcept {
uint64_t const mask = 0x000000FF000000FF;
uint64_t const mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32)
@ -153,7 +153,7 @@ parse_8_digits(uint64_t val) noexcept {
// Call this if chars are definitely 8 digits.
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint32_t
fastfloat_inline FASTFLOAT_CONSTEXPR20 uint32_t
parse_8_digits(UC const *chars) noexcept {
if (is_constant_evaluated() || !has_simd_opt<UC>()) {
return parse_8_digits(
@ -163,18 +163,16 @@ parse_8_digits(UC const *chars) noexcept {
}
// credit @aqrit
fastfloat_really_inline constexpr bool
is_made_of_8_digits(uint64_t val) noexcept {
fastfloat_inline constexpr bool is_made_of_8_digits(uint64_t val) noexcept {
return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) &
0x8080808080808080));
}
fastfloat_really_inline constexpr bool
is_made_of_4_digits(uint32_t val) noexcept {
fastfloat_inline constexpr bool is_made_of_4_digits(uint32_t val) noexcept {
return !((((val + 0x46464646) | (val - 0x30303030)) & 0x80808080));
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint32_t
fastfloat_inline FASTFLOAT_CONSTEXPR14 uint32_t
parse_4_digits(uint32_t val) noexcept {
val -= 0x30303030;
val = (val * 10) + (val >> 8);
@ -187,7 +185,7 @@ parse_4_digits(uint32_t val) noexcept {
#if FASTFLOAT_X86_SIMD >= 31
// credit @hedgehoginthecpp
fastfloat_really_inline __m128i parse_4x4_digits(__m128i data) noexcept {
fastfloat_inline __m128i parse_4x4_digits(__m128i data) noexcept {
// 1. convert from ASCII '0' .. '9' to numbers 0 .. 9
const __m128i ascii0 = _mm_set1_epi8('0');
const __m128i t0 = _mm_subs_epu8(data, ascii0);
@ -204,8 +202,7 @@ fastfloat_really_inline __m128i parse_4x4_digits(__m128i data) noexcept {
#endif
// credit @hedgehoginthecpp
fastfloat_really_inline uint64_t
convert_4x4_to_16_digits(__m128i data) noexcept {
fastfloat_inline uint64_t convert_4x4_to_16_digits(__m128i data) noexcept {
// 4. convert to 16-digit number
// v[0] * 10^12 + v[1] * 10^8 + v[2] * 10^4 + v[3]
const uint64_t lo = static_cast<uint32_t>(_mm_cvtsi128_si32(data));
@ -222,8 +219,8 @@ convert_4x4_to_16_digits(__m128i data) noexcept {
#if FASTFLOAT_X86_SIMD >= 42
// credit @hedgehoginthecpp
fastfloat_really_inline bool parse_if_16_digits(char const *chars,
uint64_t &value) noexcept {
fastfloat_inline bool parse_if_16_digits(char const *chars,
uint64_t &value) noexcept {
FASTFLOAT_SIMD_DISABLE_WARNINGS
const __m128i data =
_mm_loadu_si128(reinterpret_cast<__m128i const *>(chars));
@ -262,14 +259,14 @@ fastfloat_really_inline bool parse_if_16_digits(char const *chars,
#if FASTFLOAT_X86_SIMD >= 31
// credit @hedgehoginthecpp
fastfloat_really_inline uint64_t parse_16_digits(char const *p) noexcept {
fastfloat_inline uint64_t parse_16_digits(char const *p) noexcept {
const __m128i data = _mm_loadu_si128(reinterpret_cast<const __m128i *>(p));
return convert_4x4_to_16_digits(parse_4x4_digits(data));
}
#endif
// credit @hedgehoginthecpp
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
fastfloat_inline FASTFLOAT_CONSTEXPR20 void
parse_digits_until_19(char const *&p, char const *pend, am_mant_t &mantissa) {
#if FASTFLOAT_X86_SIMD >= 31
if (!is_constant_evaluated()) {
@ -302,7 +299,7 @@ parse_digits_until_19(char const *&p, char const *pend, am_mant_t &mantissa) {
}
template <typename UC, FASTFLOAT_ENABLE_IF(!std::is_same<UC, char>::value) = 0>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
fastfloat_inline FASTFLOAT_CONSTEXPR20 void
parse_digits_until_19(UC const *&p, UC const *pend,
am_mant_t &mantissa) noexcept {
do {
@ -311,7 +308,7 @@ parse_digits_until_19(UC const *&p, UC const *pend,
}
#else
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
fastfloat_inline FASTFLOAT_CONSTEXPR20 void
parse_digits_until_19(UC const *&p, UC const *pend,
am_mant_t &mantissa) noexcept {
do {
@ -323,7 +320,7 @@ parse_digits_until_19(UC const *&p, UC const *pend,
// Call this if chars might not be 8 digits.
// Using this style (instead of is_made_of_8_digits() then
// parse_8_digits()) ensures we don't load SIMD registers twice.
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool
simd_parse_if_8_digits(char16_t const *chars, uint64_t &i) noexcept {
if (is_constant_evaluated()) {
return false;
@ -370,7 +367,7 @@ simd_parse_if_8_digits(char16_t const *chars, uint64_t &i) noexcept {
}
#else
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
fastfloat_inline FASTFLOAT_CONSTEXPR20 void
parse_digits_until_19(UC const *&p, UC const *pend,
am_mant_t &mantissa) noexcept {
do {
@ -391,7 +388,7 @@ constexpr bool simd_parse_if_8_digits(UC const *, uint64_t &) {
}
template <typename UC, FASTFLOAT_ENABLE_IF(!std::is_same<UC, char>::value) = 0>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
fastfloat_inline FASTFLOAT_CONSTEXPR20 void
loop_parse_if_digits(UC const *&p, UC const *const pend, uint64_t &i) noexcept {
if (!is_constant_evaluated()) {
if FASTFLOAT_CONSTEXPR17 (has_simd_opt<UC>()) {
@ -408,7 +405,7 @@ loop_parse_if_digits(UC const *&p, UC const *const pend, uint64_t &i) noexcept {
}
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
fastfloat_inline FASTFLOAT_CONSTEXPR20 void
loop_parse_if_digits(char const *&p, char const *const pend,
uint64_t &i) noexcept {
#if FASTFLOAT_USE_SIMD && FASTFLOAT_X86_SIMD >= 42
@ -499,7 +496,7 @@ using parsed_number_string = parsed_number_string_t<char>;
// Helper for error creating
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t<UC> &
fastfloat_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t<UC> &
report_parse_error(parsed_number_string_t<UC> &answer, UC const *p,
parse_error error) noexcept {
answer.invalid = true;
@ -517,11 +514,11 @@ report_parse_error(parsed_number_string_t<UC> &answer, UC const *p,
// spans (read only by the rare digit_comp slow path) are not materialized,
// which keeps the fat parsed_number_string_t off the hot path. The caller
// re-parses with store_spans=true if the slow path is actually reached.
template <bool json_fmt, typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t<UC>
parse_number_string(UC const *p, UC const *pend,
parse_options_t<UC> const options,
bool store_spans = true) noexcept {
template <bool json_fmt, bool javascript_fmt, typename UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t<UC>
parse_number_string_impl(UC const *p, UC const *pend,
parse_options_t<UC> const options,
bool const store_spans) noexcept {
parsed_number_string_t<UC> answer{};
FASTFLOAT_ASSUME(p < pend); // so dereference without checks
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
@ -555,7 +552,7 @@ parse_number_string(UC const *p, UC const *pend,
// HedgehogInTheCPP: compiler generate much better code in any mode without
// manual unroling because it's less branching and also it's allow more
// inlining which is better. A a multiplication by 10 is cheaper than an
// inlining which is better. A multiplication by 10 is cheaper than an
// arbitrary integer multiplication. might overflow, handled later
#if !defined(FASTFLOAT_ISNOT_CHECKED_BOUNDS) && \
!defined(FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN)
@ -584,7 +581,7 @@ parse_number_string(UC const *p, UC const *pend,
return report_parse_error<UC>(answer, p,
parse_error::no_digits_in_integer_part);
}
if (json_fmt || chars_format_t(options.format & detail::javascript_fmt)) {
if (json_fmt || javascript_fmt) {
// ECMAScript DecimalIntegerLiteral is "0" or a non-zero digit followed by
// digits: no leading zeros. Unlike JSON, the integer part may be empty
// (".5"); the no_digits_in_mantissa check below still rejects ".".
@ -595,7 +592,7 @@ parse_number_string(UC const *p, UC const *pend,
return report_parse_error<UC>(
answer, start_digits, parse_error::leading_zeros_in_integer_part);
}
if (!json_fmt) {
if (javascript_fmt) {
// Sloppy mode (Annex B): a NonOctalDecimalIntegerLiteral has a leading
// zero and at least one digit that is 8 or 9 ("08.5" is 8.5). With
// octal digits only, it is a LegacyOctalIntegerLiteral ("0775"), which
@ -765,8 +762,45 @@ parse_number_string(UC const *p, UC const *pend,
return answer;
}
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
// Cold instantiation of the parser: the ECMAScript integer-part rule costs two
// error returns plus the Annex B octal scan, and parse_number_string_impl is
// force-inlined, so an inlined javascript body would enlarge the frame of every
// caller that never asks for it. Out of line, it costs those callers nothing.
template <typename UC>
fastfloat_noinline FASTFLOAT_CONSTEXPR20 parsed_number_string_t<UC>
parse_number_string_javascript(UC const *p, UC const *pend,
parse_options_t<UC> const options,
bool const store_spans) noexcept {
return parse_number_string_impl<false, true, UC>(p, pend, options,
store_spans);
}
// Public entry point, behaviour unchanged: chars_format::javascript is still
// honoured, it is just selected here once instead of being re-tested inside the
// parser loop. Callers that have already ruled the format out (see
// from_chars_float_advanced) should call parse_number_string_impl directly so
// that not even this test reaches their hot path.
template <bool basic_json_fmt, typename UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t<UC>
parse_number_string(UC const *p, UC const *pend,
parse_options_t<UC> const options,
bool const store_spans = true) noexcept {
// JSON and JavaScript are mutually exclusive, so only the non-JSON
// instantiation has to look at the flag.
if (!basic_json_fmt) {
if fastfloat_unlikely (chars_format_t(options.format &
detail::basic_javascript_fmt)) {
return parse_number_string_javascript<UC>(p, pend, options, store_spans);
}
}
return parse_number_string_impl<basic_json_fmt, false, UC>(p, pend, options,
store_spans);
}
#endif
template <typename T, typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
parse_int_string(UC const *p, UC const *pend, T &value,
parse_options_t<UC> const options) noexcept {
FASTFLOAT_ASSUME(p < pend); // so dereference without checks

40
include/fast_float/bigint.h
View File

@ -158,20 +158,20 @@ template <limb_t size> struct stackvec {
}
};
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 uint64_t
fastfloat_inline FASTFLOAT_CONSTEXPR14 uint64_t
empty_hi64(bool &truncated) noexcept {
truncated = false;
return 0;
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t
fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t
uint64_hi64(uint64_t r0, bool &truncated) noexcept {
truncated = false;
auto const shl = leading_zeroes(r0);
return r0 << shl;
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t
fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t
uint64_hi64(uint64_t r0, uint64_t r1, bool &truncated) noexcept {
auto const shl = leading_zeroes(r0);
if (shl == 0) {
@ -184,19 +184,19 @@ uint64_hi64(uint64_t r0, uint64_t r1, bool &truncated) noexcept {
}
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t
fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t
uint32_hi64(uint32_t r0, bool &truncated) noexcept {
return uint64_hi64(r0, truncated);
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t
fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t
uint32_hi64(uint32_t r0, uint32_t r1, bool &truncated) noexcept {
uint64_t x0 = r0;
uint64_t x1 = r1;
return uint64_hi64((x0 << 32) | x1, truncated);
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t
fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t
uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool &truncated) noexcept {
uint64_t x0 = r0;
uint64_t x1 = r1;
@ -208,7 +208,7 @@ uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool &truncated) noexcept {
// we want an efficient operation. for msvc, where
// we don't have built-in intrinsics, this is still
// pretty fast.
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb
fastfloat_inline FASTFLOAT_CONSTEXPR20 limb
scalar_add(limb x, limb y, bool &overflow) noexcept {
limb z;
// gcc and clang
@ -226,8 +226,8 @@ scalar_add(limb x, limb y, bool &overflow) noexcept {
}
// multiply two small integers, getting both the high and low bits.
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb
scalar_mul(limb x, limb y, limb &carry) noexcept {
fastfloat_inline FASTFLOAT_CONSTEXPR20 limb scalar_mul(limb x, limb y,
limb &carry) noexcept {
#ifdef FASTFLOAT_64BIT_LIMB
#if defined(__SIZEOF_INT128__)
// GCC and clang both define it as an extension.
@ -254,7 +254,7 @@ scalar_mul(limb x, limb y, limb &carry) noexcept {
// add scalar value to bigint starting from offset.
// used in grade school multiplication
template <limb_t size>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool
small_add_from(stackvec<size> &vec, limb carry, limb_t start) noexcept {
bool overflow;
while (carry != 0 && start < vec.len()) {
@ -270,15 +270,15 @@ small_add_from(stackvec<size> &vec, limb carry, limb_t start) noexcept {
// add scalar value to bigint.
template <limb_t size>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
small_add(stackvec<size> &vec, limb y) noexcept {
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool small_add(stackvec<size> &vec,
limb y) noexcept {
return small_add_from(vec, y, 0);
}
// multiply bigint by scalar value.
template <limb_t size>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
small_mul(stackvec<size> &vec, limb y) noexcept {
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool small_mul(stackvec<size> &vec,
limb y) noexcept {
limb carry = 0;
for (limb_t index = 0; index != vec.len(); ++index) {
vec[index] = scalar_mul(vec[index], y, carry);
@ -292,7 +292,7 @@ small_mul(stackvec<size> &vec, limb y) noexcept {
// add bigint to bigint starting from index.
// used in grade school multiplication
template <limb_t size>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool
large_add_from(stackvec<size> &x, limb_span y, limb_t start) noexcept {
// the effective x buffer is from `xstart..x.len()`, so exit early
// if we can't get that current range.
@ -324,15 +324,15 @@ large_add_from(stackvec<size> &x, limb_span y, limb_t start) noexcept {
// add bigint to bigint.
template <limb_t size>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool
large_add_from(stackvec<size> &x, limb_span y) noexcept {
return large_add_from(x, y, 0);
}
// grade-school multiplication algorithm
template <limb_t size>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
long_mul(stackvec<size> &x, limb_span y) noexcept {
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool long_mul(stackvec<size> &x,
limb_span y) noexcept {
limb_span xs = limb_span(x.data, x.len());
stackvec<size> z(xs);
limb_span zs = limb_span(z.data, z.len());
@ -360,8 +360,8 @@ long_mul(stackvec<size> &x, limb_span y) noexcept {
// grade-school multiplication algorithm
template <limb_t size>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
large_mul(stackvec<size> &x, limb_span y) noexcept {
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool large_mul(stackvec<size> &x,
limb_span y) noexcept {
if (y.len() == 1) {
FASTFLOAT_TRY(small_mul(x, y[0]));
} else {

70
include/fast_float/decimal_to_binary.h
View File

@ -18,7 +18,7 @@ namespace fast_float {
// bits.
//
template <am_bits_t bit_precision>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 value128
fastfloat_inline FASTFLOAT_CONSTEXPR20 value128
compute_product_approximation(am_pow_t q, am_mant_t w) noexcept {
am_pow_t const index = 2 * (q - powers::smallest_power_of_five);
// For small values of q, e.g., q in [0,27], the answer is always exact
@ -64,7 +64,7 @@ namespace detail {
* where
* p = log(5**-q)/log(2) = -q * log(5)/log(2)
*/
constexpr fastfloat_really_inline am_pow_t power(am_pow_t const q) noexcept {
constexpr fastfloat_inline am_pow_t power(am_pow_t const q) noexcept {
return (((152170 + 65536) * q) >> 16) + 63;
}
} // namespace detail
@ -72,7 +72,7 @@ constexpr fastfloat_really_inline am_pow_t power(am_pow_t const q) noexcept {
// create an adjusted mantissa, biased by the invalid power2
// for significant digits already multiplied by 10 ** q.
template <typename binary>
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 adjusted_mantissa
fastfloat_inline FASTFLOAT_CONSTEXPR14 adjusted_mantissa
compute_error_scaled(am_pow_t q, am_mant_t w, limb_t lz) noexcept {
auto const hilz = static_cast<am_bits_t>((w >> 63) ^ 1);
adjusted_mantissa answer;
@ -86,7 +86,7 @@ compute_error_scaled(am_pow_t q, am_mant_t w, limb_t lz) noexcept {
// w * 10 ** q, without rounding the representation up.
// the power2 in the exponent will be adjusted by invalid_am_bias.
template <typename binary>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
compute_error(am_pow_t q, am_mant_t w) noexcept {
auto const lz = leading_zeroes(w);
w <<= lz;
@ -101,17 +101,17 @@ compute_error(am_pow_t q, am_mant_t w) noexcept {
// cases, we return an adjusted_mantissa with a negative power of 2: the caller
// should recompute in such cases.
template <typename binary>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
compute_float(am_pow_t q, am_mant_t w) noexcept {
adjusted_mantissa answer;
if ((w == 0) || (q < binary::smallest_power_of_ten())) {
// we want to get zero:
// result should be zero
answer.power2 = 0;
answer.mantissa = 0;
return answer;
}
if (q > binary::largest_power_of_ten()) {
// we want to get infinity:
// result should be infinity
answer.power2 = binary::infinite_power();
answer.mantissa = 0;
return answer;
@ -152,22 +152,32 @@ compute_float(am_pow_t q, am_mant_t w) noexcept {
// Now, we need to round the mantissa correctly.
if (answer.power2 <= 0) { // we have a subnormal or very small value.
if (answer.power2 <= 0) {
// We have a subnormal or very small number.
// Here have that answer.power2 <= 0 so -answer.power2 >= 0
if (-answer.power2 + 1 >=
64) { // if we have more than 64 bits below the minimum exponent, you
// have a zero for sure.
auto const subnormal_shift = static_cast<am_bits_t>(-answer.power2 + 1);
if (subnormal_shift >= 64) {
// We have more than 64 bits below the minimum exponent
// result should be zero
answer.power2 = 0;
answer.mantissa = 0;
// result should be zero
return answer;
}
// We have a subnormal number. We need to shift the mantissa to the right
// next line is safe because -answer.power2 + 1 < 64
answer.mantissa >>= -answer.power2 + 1;
// Thankfully, we can't have both "round-to-even" and subnormals because
// "round-to-even" only occurs for powers close to 0 in the 32-bit and
// and 64-bit case (with no more than 19 digits).
answer.mantissa >>= subnormal_shift;
// A subnormal result can also fall exactly between two floats. With at
// most 19 digits this never happens for float and double, but it does for
// std::float16_t (e.g., 2^-25 = 298023223876953125e-25), so we apply the
// same round-to-even test as in the normal case below.
// See script/format_parameters.py.
if (binary::subnormal_ties_possible() && (product.low <= 1) &&
(q >= binary::min_exponent_round_to_even()) &&
(q <= binary::max_exponent_round_to_even()) &&
((answer.mantissa & 3) == 1)) {
if (((answer.mantissa << subnormal_shift) << shift) == product.high) {
answer.mantissa &= ~am_mant_t(1); // flip it so that we do not round up
}
}
answer.mantissa += (answer.mantissa & 1); // round up
answer.mantissa >>= 1;
// There is a weird scenario where we don't have a subnormal but just.
@ -184,8 +194,8 @@ compute_float(am_pow_t q, am_mant_t w) noexcept {
return answer;
}
// usually, we round *up*, but if we fall right in between and and we have an
// even basis, we need to round down
// Usually, we round *up*, but if we fall right in between and we have an even
// basis, we need to round down.
// We are only concerned with the cases where 5**q fits in single 64-bit word.
if ((product.low <= 1) && (q >= binary::min_exponent_round_to_even()) &&
(q <= binary::max_exponent_round_to_even()) &&
@ -203,18 +213,30 @@ compute_float(am_pow_t q, am_mant_t w) noexcept {
// Normal rounding
answer.mantissa += (answer.mantissa & 1); // round up
answer.mantissa >>= 1;
if (answer.mantissa >= (am_mant_t(2) << binary::mantissa_explicit_bits())) {
// Both fix-ups below are rare. They are marked unlikely so that clang keeps
// them as branches instead of folding them into conditional moves that
// every conversion pays for.
#ifdef __clang__
#pragma clang diagnostic push
#if (!defined(__APPLE_CC__) && __clang_major__ >= 10) || (__clang_major__ >= 13)
#pragma clang diagnostic ignored "-Wc++20-extensions"
#endif
#endif
if fastfloat_unlikely (answer.mantissa >=
(am_mant_t(2) << binary::mantissa_explicit_bits())) {
answer.mantissa = (am_mant_t(1) << binary::mantissa_explicit_bits());
++answer.power2; // undo previous line addition
++answer.power2; // undo previous addition
}
// Check if we have infinity after computation
answer.mantissa &= ~(am_mant_t(1) << binary::mantissa_explicit_bits());
if (answer.power2 >= binary::infinite_power()) { // infinity
if fastfloat_unlikely (answer.power2 >= binary::infinite_power()) {
// result should be infinity
answer.power2 = binary::infinite_power();
answer.mantissa = 0;
}
#ifdef __clang__
#pragma clang diagnostic pop
#endif
return answer;
}

41
include/fast_float/digit_comparison.h
View File

@ -37,7 +37,7 @@ constexpr static uint64_t powers_of_ten_uint64[] = {1UL,
// this algorithm is not even close to optimized, but it has no practical
// effect on performance: in order to have a faster algorithm, we'd need
// to slow down performance for faster algorithms, and this is still fast.
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 am_pow_t
fastfloat_inline FASTFLOAT_CONSTEXPR14 am_pow_t
scientific_exponent(am_mant_t mantissa, am_pow_t exponent) noexcept {
while (mantissa >= 10000) {
mantissa /= 10000;
@ -56,7 +56,7 @@ scientific_exponent(am_mant_t mantissa, am_pow_t exponent) noexcept {
// this converts a native floating-point number to an extended-precision float.
template <typename T>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
to_extended(T const value) noexcept {
using equiv_uint = equiv_uint_t<T>;
constexpr equiv_uint exponent_mask = binary_format<T>::exponent_mask();
@ -88,7 +88,7 @@ to_extended(T const value) noexcept {
// we are given a native float that represents b, so we need to adjust it
// halfway between b and b+u.
template <typename T>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
to_extended_halfway(T const value) noexcept {
adjusted_mantissa am = to_extended(value);
am.mantissa <<= 1;
@ -99,8 +99,8 @@ to_extended_halfway(T const value) noexcept {
// round an extended-precision float to the nearest machine float.
template <typename T, typename callback>
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am,
callback cb) noexcept {
fastfloat_inline FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am,
callback cb) noexcept {
constexpr am_pow_t mantissa_shift =
64 - binary_format<T>::mantissa_explicit_bits() - 1;
if (-am.power2 >= mantissa_shift) {
@ -134,7 +134,7 @@ fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am,
}
template <typename callback>
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void
fastfloat_inline FASTFLOAT_CONSTEXPR14 void
round_nearest_tie_even(adjusted_mantissa &am, am_pow_t shift,
callback cb) noexcept {
am_mant_t const mask = (shift == 64) ? std::numeric_limits<am_mant_t>::max()
@ -156,7 +156,7 @@ round_nearest_tie_even(adjusted_mantissa &am, am_pow_t shift,
am.mantissa += am_mant_t(cb(is_odd, is_halfway, is_above));
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void
fastfloat_inline FASTFLOAT_CONSTEXPR14 void
round_down(adjusted_mantissa &am, am_pow_t shift) noexcept {
if (shift == 64) {
am.mantissa = 0;
@ -167,7 +167,7 @@ round_down(adjusted_mantissa &am, am_pow_t shift) noexcept {
}
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
fastfloat_inline FASTFLOAT_CONSTEXPR20 void
skip_zeros(UC const *&first, UC const *last) noexcept {
if (!is_constant_evaluated()) {
while (std::distance(first, last) >= int_cmp_len<UC>()) {
@ -190,7 +190,7 @@ skip_zeros(UC const *&first, UC const *last) noexcept {
// determine if any non-zero digits were truncated.
// all characters must be valid digits.
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool
is_truncated(UC const *first, UC const *last) noexcept {
// do 8-bit optimizations, can just compare to 8 literal 0s.
if (!is_constant_evaluated()) {
@ -213,14 +213,14 @@ is_truncated(UC const *first, UC const *last) noexcept {
}
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool
is_truncated(span<UC const> s) noexcept {
return is_truncated(s.ptr, s.ptr + s.len());
}
#if defined(FASTFLOAT_64BIT_LIMB) && defined(FASTFLOAT_X86_SIMD) && \
FASTFLOAT_X86_SIMD >= 31
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
fastfloat_inline FASTFLOAT_CONSTEXPR20 void
parse_sixteen_digits(char const *&p, limb &value, am_digits &counter,
am_digits &count) noexcept {
value = parse_16_digits(p);
@ -231,7 +231,7 @@ parse_sixteen_digits(char const *&p, limb &value, am_digits &counter,
#endif
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
fastfloat_inline FASTFLOAT_CONSTEXPR20 void
parse_eight_digits(UC const *&p, limb &value, am_digits &counter,
am_digits &count) noexcept {
value = value * 100000000 + parse_8_digits(p);
@ -241,7 +241,7 @@ parse_eight_digits(UC const *&p, limb &value, am_digits &counter,
}
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void
fastfloat_inline FASTFLOAT_CONSTEXPR14 void
parse_four_digits(UC const *&p, limb &value, am_digits &counter,
am_digits &count) noexcept {
value = value * 10000 + parse_4_digits(read_chars_to_unsigned<uint32_t>(p));
@ -251,7 +251,7 @@ parse_four_digits(UC const *&p, limb &value, am_digits &counter,
}
template <typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 void
fastfloat_inline FASTFLOAT_CONSTEXPR14 void
parse_one_digit(UC const *&p, limb &value, am_digits &counter,
am_digits &count) noexcept {
value = value * 10 + limb(*p - UC('0'));
@ -260,13 +260,13 @@ parse_one_digit(UC const *&p, limb &value, am_digits &counter,
++count;
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
add_native(bigint &big, limb power, limb value) noexcept {
fastfloat_inline FASTFLOAT_CONSTEXPR20 void add_native(bigint &big, limb power,
limb value) noexcept {
big.mul(power);
big.add(value);
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void
fastfloat_inline FASTFLOAT_CONSTEXPR20 void
round_up_bigint(bigint &big, am_digits &count) noexcept {
// need to round-up the digits, but need to avoid rounding
// ....9999 to ...10000, which could cause a false halfway point.
@ -388,9 +388,8 @@ parse_mantissa(bigint &result, const parsed_number_string_t<UC> &num) noexcept {
}
template <typename T>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
positive_digit_comp(bigint &bigmant, adjusted_mantissa am,
am_pow_t const exponent) noexcept {
fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa positive_digit_comp(
bigint &bigmant, adjusted_mantissa am, am_pow_t const exponent) noexcept {
FASTFLOAT_ASSERT(bigmant.pow10(exponent));
bool truncated;
am.mantissa = bigmant.hi64(truncated);
@ -417,7 +416,7 @@ positive_digit_comp(bigint &bigmant, adjusted_mantissa am,
// we then need to scale by `2^(f- e)`, and then the two significant digits
// are of the same magnitude.
template <typename T>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
fastfloat_inline FASTFLOAT_CONSTEXPR20 adjusted_mantissa
negative_digit_comp(bigint &real_digits, adjusted_mantissa am,
am_pow_t const real_exp) noexcept {
// get the value of `b`, rounded down, and get a bigint representation of b+h

224
include/fast_float/float_common.h
View File

@ -254,13 +254,13 @@ static_assert(FASTFLOAT_X86_SIMD == 20 || FASTFLOAT_X86_SIMD == 42 ||
#endif
#ifdef FASTFLOAT_VISUAL_STUDIO
#define fastfloat_really_inline __forceinline
#define fastfloat_inline __forceinline
#define fastfloat_noinline __declspec(noinline)
#elif defined(__GNUC__) || defined(__clang__)
#define fastfloat_really_inline inline __attribute__((always_inline))
#define fastfloat_inline inline __attribute__((always_inline))
#define fastfloat_noinline __attribute__((noinline))
#else
#define fastfloat_really_inline inline
#define fastfloat_inline inline
#define fastfloat_noinline
#endif
@ -325,7 +325,7 @@ namespace fast_float {
using std::bit_cast;
#else
template <typename To, typename From>
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 To bit_cast(const From &from) {
fastfloat_inline FASTFLOAT_CONSTEXPR14 To bit_cast(const From &from) {
// Implementation of std::bit_cast for pre-C++20.
auto to = To();
// The cast suppresses a bogus -Wclass-memaccess on GCC.
@ -337,7 +337,7 @@ fastfloat_really_inline FASTFLOAT_CONSTEXPR14 To bit_cast(const From &from) {
#if FASTFLOAT_HAS_IS_CONSTANT_EVALUATED
using std::is_constant_evaluated;
#else
fastfloat_really_inline constexpr bool is_constant_evaluated() noexcept {
fastfloat_inline constexpr bool is_constant_evaluated() noexcept {
return false;
}
#endif
@ -384,135 +384,23 @@ struct is_supported_char_type
};
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
// Compares two ASCII strings in a case insensitive manner. The expected
// string is lowercase ASCII, so OR-ing 0x20 into each actual character maps
// 'A'..'Z' onto 'a'..'z' and leaves everything else mismatched. The lengths
// used are tiny constants (3 and 5), so the compiler unrolls this loop; keep
// it small so that the (rarely executed) inf/nan path does not bloat the hot
// parser it is inlined into.
template <typename UC>
inline FASTFLOAT_CONSTEXPR20 bool
fastfloat_strncasecmp3(UC const *actual_mixedcase,
UC const *expected_lowercase) {
if (is_constant_evaluated()) {
for (uint_fast8_t i = 0; i != 3; ++i) {
if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) {
return false;
}
}
return true;
} else {
uint64_t val1{0}, val2{0};
uint64_t mask{0};
if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 1) {
mask = 0x2020202020202020;
} else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 2) {
mask = 0x0020002000200020;
} else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 4) {
mask = 0x0000002000000020;
}
if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 1 || sizeof(UC) == 2) {
std::memcpy(&val1, actual_mixedcase, 3 * sizeof(UC));
std::memcpy(&val2, expected_lowercase, 3 * sizeof(UC));
val1 |= mask;
val2 |= mask;
return val1 == val2;
} else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 4) {
std::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC));
std::memcpy(&val2, expected_lowercase, 2 * sizeof(UC));
val1 |= mask;
if (val1 != val2) {
return false;
}
return (actual_mixedcase[2] | 32) == (expected_lowercase[2]);
} else {
return false;
}
}
}
template <typename UC>
inline FASTFLOAT_CONSTEXPR20 bool
fastfloat_strncasecmp5(UC const *actual_mixedcase,
UC const *expected_lowercase) noexcept {
if (is_constant_evaluated()) {
for (uint_fast8_t i = 0; i != 5; ++i) {
if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) {
return false;
}
}
return true;
} else {
uint64_t val1{0}, val2{0};
if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 1) {
constexpr uint64_t mask = 0x2020202020202020;
std::memcpy(&val1, actual_mixedcase, 5 * sizeof(UC));
std::memcpy(&val2, expected_lowercase, 5 * sizeof(UC));
val1 |= mask;
val2 |= mask;
return val1 == val2;
} else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 2) {
constexpr uint64_t mask = 0x0020002000200020;
std::memcpy(&val1, actual_mixedcase, 4 * sizeof(UC));
std::memcpy(&val2, expected_lowercase, 4 * sizeof(UC));
val1 |= mask;
if (val1 != val2) {
return false;
}
return (actual_mixedcase[4] | 32) == (expected_lowercase[4]);
} else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 4) {
constexpr uint64_t mask = 0x0000002000000020;
std::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC));
std::memcpy(&val2, expected_lowercase, 2 * sizeof(UC));
val1 |= mask;
if (val1 != val2) {
return false;
}
std::memcpy(&val1, actual_mixedcase + 2, 2 * sizeof(UC));
std::memcpy(&val2, expected_lowercase + 2, 2 * sizeof(UC));
val1 |= mask;
if (val1 != val2) {
return false;
}
return (actual_mixedcase[4] | 32) == (expected_lowercase[4]);
} else {
return false;
}
}
}
// Compares two ASCII strings in a case insensitive manner.
template <typename UC>
inline FASTFLOAT_CONSTEXPR20 bool
inline FASTFLOAT_CONSTEXPR14 bool
fastfloat_strncasecmp(UC const *actual_mixedcase, UC const *expected_lowercase,
uint_fast8_t const length) noexcept {
if (is_constant_evaluated()) {
for (uint_fast8_t i = 0; i != length; ++i) {
if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) {
return false;
}
}
return true;
} else {
uint64_t val1{0}, val2{0};
uint64_t mask{0};
if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 1) {
mask = 0x2020202020202020;
} else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 2) {
mask = 0x0020002000200020;
} else if FASTFLOAT_CONSTEXPR17 (sizeof(UC) == 4) {
mask = 0x0000002000000020;
}
uint_fast8_t sz = 8 / (sizeof(UC));
for (uint_fast8_t i = 0; i < length; i += sz) {
sz = sz < (length - i) ? sz : length - i;
std::memcpy(&val1, actual_mixedcase + i, sz * sizeof(UC));
std::memcpy(&val2, expected_lowercase + i, sz * sizeof(UC));
val1 |= mask;
val2 |= mask;
if (val1 != val2) {
return false;
}
for (uint_fast8_t i = 0; i != length; ++i) {
if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) {
return false;
}
}
return true;
}
#endif
#ifndef FLT_EVAL_METHOD
@ -548,7 +436,7 @@ struct value128 {
};
/* Helper C++14 constexpr generic implementation of leading_zeroes for 64-bit */
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 limb_t
fastfloat_inline FASTFLOAT_CONSTEXPR14 limb_t
leading_zeroes_generic(uint64_t input_num) noexcept {
assert(input_num > 0);
FASTFLOAT_ASSUME(input_num > 0);
@ -580,7 +468,7 @@ leading_zeroes_generic(uint64_t input_num) noexcept {
}
/* result might be undefined when input_num is zero */
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb_t
fastfloat_inline FASTFLOAT_CONSTEXPR20 limb_t
leading_zeroes(uint64_t input_num) noexcept {
assert(input_num > 0);
FASTFLOAT_ASSUME(input_num > 0);
@ -615,7 +503,7 @@ leading_zeroes(uint64_t input_num) noexcept {
}
/* Helper C++14 constexpr generic implementation of countr_zero for 32-bit */
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 limb_t
fastfloat_inline FASTFLOAT_CONSTEXPR14 limb_t
countr_zero_generic_32(uint32_t input_num) noexcept {
assert(input_num > 0);
FASTFLOAT_ASSUME(input_num > 0);
@ -643,7 +531,7 @@ countr_zero_generic_32(uint32_t input_num) noexcept {
}
/* count trailing zeroes for 32-bit integers */
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 limb_t
fastfloat_inline FASTFLOAT_CONSTEXPR20 limb_t
countr_zero_32(uint32_t input_num) noexcept {
if (is_constant_evaluated()) {
return countr_zero_generic_32(input_num);
@ -659,12 +547,12 @@ countr_zero_32(uint32_t input_num) noexcept {
#endif
}
fastfloat_really_inline constexpr uint64_t emulu_generic(uint32_t x,
fastfloat_inline constexpr uint64_t emulu_generic(uint32_t x,
uint32_t y) noexcept {
return x * static_cast<uint64_t>(y);
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t
fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t
umul128_generic(uint64_t ab, uint64_t cd, uint64_t &hi) noexcept {
auto ab_shifted = static_cast<uint32_t>(ab >> 32);
auto cd_shifted = static_cast<uint32_t>(cd >> 32);
@ -683,7 +571,7 @@ umul128_generic(uint64_t ab, uint64_t cd, uint64_t &hi) noexcept {
}
// Compute hi and low parts of 128-bit.
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 uint64_t
fastfloat_inline FASTFLOAT_CONSTEXPR20 uint64_t
umul128(uint64_t ab, uint64_t cd, uint64_t &hi) noexcept {
if (is_constant_evaluated()) {
return umul128_generic(ab, cd, hi);
@ -702,7 +590,7 @@ umul128(uint64_t ab, uint64_t cd, uint64_t &hi) noexcept {
}
// Compute 128-bit result.
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 value128
fastfloat_inline FASTFLOAT_CONSTEXPR20 value128
full_multiplication(uint64_t a, uint64_t b) noexcept {
value128 answer;
#if defined(FASTFLOAT_64BIT) && defined(__SIZEOF_INT128__)
@ -753,6 +641,8 @@ template <typename T> struct binary_format : binary_format_lookup_tables<T> {
static constexpr am_mant_t max_mantissa_fast_path(am_pow_t const power);
static constexpr am_mant_t
max_mantissa_fast_path(); // used when fegetround() == FE_TONEAREST
static constexpr bool fast_path_can_overflow();
static constexpr bool subnormal_ties_possible();
static constexpr am_pow_t largest_power_of_ten();
static constexpr am_pow_t smallest_power_of_ten();
static constexpr T exact_power_of_ten(am_pow_t const power);
@ -784,6 +674,7 @@ template <typename U> struct binary_format_lookup_tables<double, U> {
0x20000000000000 / (constant_55555 * constant_55555 * 5),
0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5),
0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5 * 5),
0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5 * 5 * 5),
0x20000000000000 / (constant_55555 * constant_55555 * constant_55555),
0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * 5),
0x20000000000000 /
@ -799,9 +690,7 @@ template <typename U> struct binary_format_lookup_tables<double, U> {
0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 *
constant_55555 * 5 * 5),
0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 *
constant_55555 * 5 * 5 * 5),
0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 *
constant_55555 * 5 * 5 * 5 * 5)};
constant_55555 * 5 * 5 * 5)};
};
#if FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE
@ -1033,7 +922,9 @@ binary_format<std::float16_t>::max_mantissa_fast_path(am_pow_t power) {
template <>
inline constexpr am_bits_t
binary_format<std::float16_t>::min_exponent_fast_path() {
return 0;
// w / 10^k with w <= 2^11 and k <= 4 rounds correctly even when evaluated
// in float or double first (checked in script/format_parameters.py).
return -4;
}
template <>
@ -1045,7 +936,9 @@ binary_format<std::float16_t>::max_exponent_round_to_even() {
template <>
inline constexpr am_pow_t
binary_format<std::float16_t>::min_exponent_round_to_even() {
return -22;
// -22 covers the normal ties; subnormal ties such as
// 2^-25 = 298023223876953125e-25 need q = -25 and q = -26.
return -26;
}
template <>
@ -1076,7 +969,8 @@ binary_format<std::float16_t>::largest_power_of_ten() {
template <>
inline constexpr am_pow_t
binary_format<std::float16_t>::smallest_power_of_ten() {
return -27;
// (10^19 - 1) * 10^-27 < 2^-25, so any q < -26 rounds to zero.
return -26;
}
template <>
@ -1173,7 +1067,8 @@ binary_format<std::bfloat16_t>::max_mantissa_fast_path(am_pow_t power) {
template <>
inline constexpr am_bits_t
binary_format<std::bfloat16_t>::min_exponent_fast_path() {
return 0;
// Same argument as for std::float16_t (w <= 2^8, k <= 3).
return -3;
}
template <>
@ -1216,7 +1111,8 @@ binary_format<std::bfloat16_t>::largest_power_of_ten() {
template <>
inline constexpr am_pow_t
binary_format<std::bfloat16_t>::smallest_power_of_ten() {
return -60;
// (10^19 - 1) * 10^-60 < 2^-134, so any q < -59 rounds to zero.
return -59;
}
template <>
@ -1225,6 +1121,25 @@ inline constexpr am_digits binary_format<std::bfloat16_t>::max_digits() {
}
#endif // __STDCPP_BFLOAT16_T__
// Whether Clinger's fast path can overflow: only for std::float16_t, where
// 2^11 * 10^4 > 65504.
template <typename T>
inline constexpr bool binary_format<T>::fast_path_can_overflow() {
return double(max_mantissa_fast_path()) *
double(exact_power_of_ten(max_exponent_fast_path())) >
double((std::numeric_limits<T>::max)());
}
// A subnormal needs w * 10^q < 2^(minimum_exponent() + 1), so q is at most
// (minimum_exponent() + 1) * log10(2), with 1233/4096 < log10(2). Only
// std::float16_t has such q in its round-to-even range. We compare
// 4096 * q with (minimum_exponent() + 1) * 1233 to avoid right-shifting a
// negative value (implementation-defined before C++20).
template <typename T>
inline constexpr bool binary_format<T>::subnormal_ties_possible() {
return min_exponent_round_to_even() * 4096 <= (minimum_exponent() + 1) * 1233;
}
template <>
inline constexpr am_mant_t
binary_format<double>::max_mantissa_fast_path(am_pow_t power) {
@ -1342,7 +1257,7 @@ binary_format<double>::hidden_bit_mask() {
}
template <typename T>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void to_float(
fastfloat_inline FASTFLOAT_CONSTEXPR20 void to_float(
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
bool const negative,
#endif
@ -1502,7 +1417,7 @@ template <typename T> constexpr uint64_t int_luts<T>::min_safe_u64[];
#endif
template <typename UC>
fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) noexcept {
fastfloat_inline constexpr uint8_t ch_to_digit(UC c) noexcept {
// wchar_t and char can be signed, so we need to be careful.
using UnsignedUC = typename std::make_unsigned<UC>::type;
return int_luts<>::chdigit[static_cast<unsigned char>(
@ -1511,13 +1426,13 @@ fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) noexcept {
-((static_cast<UnsignedUC>(c) & ~0xFFull) == 0)))];
}
fastfloat_really_inline constexpr limb_t max_digits_u64(base_t base) noexcept {
fastfloat_inline constexpr limb_t max_digits_u64(base_t base) noexcept {
return int_luts<>::maxdigits_u64[base - 2];
}
// If a u64 is exactly max_digits_u64() in length, this is
// the value below which it has definitely overflowed.
fastfloat_really_inline constexpr uint64_t min_safe_u64(base_t base) noexcept {
fastfloat_inline constexpr uint64_t min_safe_u64(base_t base) noexcept {
return int_luts<>::min_safe_u64[base - 2];
}
@ -1594,21 +1509,6 @@ constexpr chars_format operator^(chars_format lhs, chars_format rhs) noexcept {
static_cast<int_type>(rhs));
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 chars_format &
operator&=(chars_format &lhs, chars_format rhs) noexcept {
return lhs = (lhs & rhs);
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 chars_format &
operator|=(chars_format &lhs, chars_format rhs) noexcept {
return lhs = (lhs | rhs);
}
fastfloat_really_inline FASTFLOAT_CONSTEXPR14 chars_format &
operator^=(chars_format &lhs, chars_format rhs) noexcept {
return lhs = (lhs ^ rhs);
}
} // namespace fast_float
#endif

160
include/fast_float/parse_number.h
View File

@ -38,7 +38,7 @@ from_chars_result_t<UC>
}
if (last - first >= 3) {
if (fastfloat_strncasecmp3(first, str_const_nan<UC>())) {
if (fastfloat_strncasecmp(first, str_const_nan<UC>(), 3)) {
answer.ptr = (first += 3);
value = minusSign ? -std::numeric_limits<T>::quiet_NaN()
: std::numeric_limits<T>::quiet_NaN();
@ -58,9 +58,9 @@ from_chars_result_t<UC>
answer.ec = std::errc();
return answer;
}
if (fastfloat_strncasecmp3(first, str_const_inf<UC>())) {
if (fastfloat_strncasecmp(first, str_const_inf<UC>(), 3)) {
if ((last - first >= 8) &&
fastfloat_strncasecmp5(first + 3, str_const_inf<UC>() + 3)) {
fastfloat_strncasecmp(first + 3, str_const_inf<UC>() + 3, 5)) {
answer.ptr = first + 8;
} else {
answer.ptr = first + 3;
@ -82,7 +82,7 @@ from_chars_result_t<UC>
* It is the default on most system. This function is meant to be inexpensive.
* Credit : @mwalcott3
*/
fastfloat_really_inline bool rounds_to_nearest() noexcept {
fastfloat_inline bool rounds_to_nearest() noexcept {
// https://lemire.me/blog/2020/06/26/gcc-not-nearest/
#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0)
return false;
@ -147,7 +147,7 @@ fastfloat_really_inline bool rounds_to_nearest() noexcept {
template <typename T> struct from_chars_caller {
template <typename UC>
FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
call(UC const *first, UC const *last, T &value,
parse_options_t<UC> const options) noexcept {
return from_chars_advanced(first, last, value, options);
@ -157,7 +157,7 @@ template <typename T> struct from_chars_caller {
#ifdef __STDCPP_FLOAT32_T__
template <> struct from_chars_caller<std::float32_t> {
template <typename UC>
FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
call(UC const *first, UC const *last, std::float32_t &value,
parse_options_t<UC> const options) noexcept {
// if std::float32_t is defined, and we are in C++23 mode; macro set for
@ -171,7 +171,7 @@ template <> struct from_chars_caller<std::float32_t> {
#ifdef __STDCPP_FLOAT64_T__
template <> struct from_chars_caller<std::float64_t> {
template <typename UC>
FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
call(UC const *first, UC const *last, std::float64_t &value,
parse_options_t<UC> const options) noexcept {
// if std::float64_t is defined, and we are in C++23 mode; macro set for
@ -182,16 +182,31 @@ template <> struct from_chars_caller<std::float64_t> {
};
#endif
// Parser instantiated for a format fixed at compile time, so that every test
// on the format folds away inside it. GCC gets the same effect by inlining the
// whole parser into each caller, where the format is a constant; clang keeps
// it out of line and would otherwise re-test each format flag per conversion.
template <typename T, typename UC, chars_format Fmt>
fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
from_chars_fixed_format(UC const *first, UC const *last, T &value) noexcept {
return from_chars_caller<T>::call(first, last, value,
parse_options_t<UC>(Fmt));
}
template <typename T, typename UC, typename>
FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
from_chars(UC const *first, UC const *last, T &value,
chars_format const fmt /*= chars_format::general*/) noexcept {
if (fmt == chars_format::general) {
return from_chars_fixed_format<T, UC, chars_format::general>(first, last,
value);
}
return from_chars_caller<T>::call(first, last, value,
parse_options_t<UC>(fmt));
}
template <typename T>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 bool
fastfloat_inline FASTFLOAT_CONSTEXPR20 bool
clinger_fast_path_impl(am_mant_t const mantissa, am_pow_t const exponent,
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
bool const is_negative,
@ -225,21 +240,25 @@ clinger_fast_path_impl(am_mant_t const mantissa, am_pow_t const exponent,
) {
// We have that fegetround() == FE_TONEAREST.
// Next is Clinger's fast path.
if (mantissa <= binary_format<T>::max_mantissa_fast_path()) {
value = static_cast<T>(mantissa);
if (exponent < 0) {
value = value / binary_format<T>::exact_power_of_ten(-exponent);
} else {
value = value * binary_format<T>::exact_power_of_ten(exponent);
value = static_cast<T>(mantissa);
if (exponent < 0) {
value = value / binary_format<T>::exact_power_of_ten(-exponent);
} else {
value = value * binary_format<T>::exact_power_of_ten(exponent);
// Only std::float16_t can overflow here (e.g., "656e2"); let the
// slow path report result_out_of_range.
if (binary_format<T>::fast_path_can_overflow() &&
value > (std::numeric_limits<T>::max)()) {
return false;
}
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
if (is_negative) {
value = -value;
}
#endif
return true;
#ifndef FASTFLOAT_ONLY_ROUNDS_TO_NEAREST_SUPPORTED
}
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
if (is_negative) {
value = -value;
}
#endif
return true;
#ifndef FASTFLOAT_ONLY_ROUNDS_TO_NEAREST_SUPPORTED
} else {
// We do not have that fegetround() == FE_TONEAREST.
// Next is a modified Clinger's fast path, inspired by Jakub Jelínek's
@ -265,8 +284,8 @@ clinger_fast_path_impl(am_mant_t const mantissa, am_pow_t const exponent,
}
#endif
return true;
#endif
}
#endif
}
}
return false;
@ -278,7 +297,7 @@ clinger_fast_path_impl(am_mant_t const mantissa, am_pow_t const exponent,
* parsing options or other parsing custom function implemented by user.
*/
template <typename T, typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
from_chars_advanced(parsed_number_string_t<UC> const &pns, T &value) noexcept {
static_assert(is_supported_float_type<T>::value,
"this type of floating-point type isn't supported");
@ -341,14 +360,50 @@ parse_number_slow_path(UC const *first, UC const *last, T &value,
) noexcept {
parsed_number_string_t<UC> const pns =
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
bjf ? parse_number_string<true, UC>(first, last, options, true) :
bjf ? parse_number_string_impl<true, false, UC>(first, last, options,
true)
:
#endif
parse_number_string<false, UC>(first, last, options, true);
parse_number_string_impl<false, false, UC>(first, last, options,
true);
return from_chars_advanced(pns, value);
}
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
// Cold: the whole chars_format::javascript conversion. Kept out of line and out
// of from_chars_float_advanced so the common frame holds neither the javascript
// parser body nor a call that would force the parsed number string onto the
// stack. Mirrors the main path, only with the javascript parser.
template <typename T, typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
fastfloat_noinline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
from_chars_float_javascript(UC const *first, UC const *last, T &value,
parse_options_t<UC> options) noexcept {
if (uint64_t(fmt & chars_format::skip_white_space)) {
while ((first != last) && fast_float::is_space(*first)) {
first++;
}
}
from_chars_result_t<UC> answer;
answer.ec = std::errc::invalid_argument;
answer.ptr = first;
if (first == last) {
return answer;
}
parsed_number_string_t<UC> pns =
parse_number_string_javascript<UC>(first, last, options, true);
if (pns.invalid) {
if (uint64_t(fmt & chars_format::no_infnan)) {
return answer;
}
return detail::parse_infnan(first, last, value, fmt);
}
return from_chars_advanced(pns, value);
}
#endif
template <typename T, typename UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
from_chars_float_advanced(UC const *first, UC const *last, T &value,
parse_options_t<UC> const options) noexcept {
static_assert(is_supported_float_type<T>::value,
@ -356,6 +411,15 @@ from_chars_float_advanced(UC const *first, UC const *last, T &value,
static_assert(is_supported_char_type<UC>::value,
"only char, wchar_t, char16_t and char32_t are supported");
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
// Leave for the javascript conversion before anything else is live, so this
// compiles to a tail call and the common path keeps its registers. The cold
// function repeats the leading-whitespace and empty-input handling below.
if fastfloat_unlikely (uint64_t(fmt & detail::basic_javascript_fmt)) {
return from_chars_float_javascript<T, UC>(first, last, value, options);
}
#endif
from_chars_result_t<UC> answer;
#ifdef FASTFLOAT_ISNOT_CHECKED_BOUNDS
// We are in parser code with external loop that checks bounds.
@ -391,12 +455,15 @@ from_chars_float_advanced(UC const *first, UC const *last, T &value,
// Fast path: parse WITHOUT materializing the integer/fraction spans (read
// only by the rare slow paths). Skipping their stores keeps the fat
// parsed_number_string_t off the hot path. store_spans is a runtime argument,
// so this reuses the single parse_number_string instantiation.
// so this reuses the single parse_number_string_impl instantiation.
parsed_number_string_t<UC> const pns =
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
bjf ? parse_number_string<true, UC>(first, last, options, false) :
bjf ? parse_number_string_impl<true, false, UC>(first, last, options,
false)
:
#endif
parse_number_string<false, UC>(first, last, options, false);
parse_number_string_impl<false, false, UC>(first, last, options,
false);
if (pns.invalid) {
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
if (chars_format_t(options.format & chars_format::no_infnan)) {
@ -458,19 +525,22 @@ from_chars_float_advanced(UC const *first, UC const *last, T &value,
#endif
);
}
#ifdef __clang__
#pragma clang diagnostic pop
#endif
to_float(
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
pns.negative,
#endif
am, value);
// Test for over/underflow.
if ((pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) ||
am.power2 == binary_format<T>::infinite_power()) {
// Test for over/underflow. Marked unlikely so that clang keeps it as a
// branch instead of folding it into a chain of conditional moves that
// every conversion pays for.
if fastfloat_unlikely ((pns.mantissa != 0 && am.mantissa == 0 &&
am.power2 == 0) ||
am.power2 == binary_format<T>::infinite_power()) {
answer.ec = std::errc::result_out_of_range;
}
#ifdef __clang__
#pragma clang diagnostic pop
#endif
return answer;
}
@ -493,15 +563,15 @@ FASTFLOAT_CONSTEXPR20
integer_times_pow10(uint64_t const mantissa,
int const decimal_exponent) noexcept {
T value;
const auto exponent = static_cast<am_pow_t>(decimal_exponent);
if (clinger_fast_path_impl(mantissa, exponent,
if (clinger_fast_path_impl(mantissa, static_cast<am_pow_t>(decimal_exponent),
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
false,
#endif
value))
return value;
adjusted_mantissa am = compute_float<binary_format<T>>(exponent, mantissa);
adjusted_mantissa am = compute_float<binary_format<T>>(
static_cast<am_pow_t>(decimal_exponent), mantissa);
to_float(
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
false,
@ -522,16 +592,16 @@ FASTFLOAT_CONSTEXPR20
FASTFLOAT_ASSUME(mantissa >= 0);
const auto m = static_cast<am_mant_t>(mantissa);
#endif
const auto exponent = static_cast<am_pow_t>(decimal_exponent);
T value;
if (clinger_fast_path_impl(m, exponent,
if (clinger_fast_path_impl(m, static_cast<am_pow_t>(decimal_exponent),
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
is_negative,
#endif
value))
return value;
adjusted_mantissa const am = compute_float<binary_format<T>>(exponent, m);
adjusted_mantissa const am = compute_float<binary_format<T>>(
static_cast<am_pow_t>(decimal_exponent), m);
to_float(
#ifndef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN
@ -634,7 +704,7 @@ template <size_t TypeIx> struct from_chars_advanced_caller {
template <> struct from_chars_advanced_caller<1> {
template <typename T, typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
call(UC const *first, UC const *last, T &value,
parse_options_t<UC> const options) noexcept {
return from_chars_float_advanced(first, last, value, options);
@ -643,7 +713,7 @@ template <> struct from_chars_advanced_caller<1> {
template <> struct from_chars_advanced_caller<2> {
template <typename T, typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 static from_chars_result_t<UC>
call(UC const *first, UC const *last, T &value,
parse_options_t<UC> const options) noexcept {
return from_chars_int_advanced(first, last, value, options);
@ -651,7 +721,7 @@ template <> struct from_chars_advanced_caller<2> {
};
template <typename T, typename UC>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
fastfloat_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t<UC>
from_chars_advanced(UC const *first, UC const *last, T &value,
parse_options_t<UC> const options) noexcept {
return from_chars_advanced_caller<

416
script/format_parameters.py Normal file
View File

@ -0,0 +1,416 @@
#
# Checks the binary_format<T> constants in include/fast_float/float_common.h
# with exact integer arithmetic, for double, float, std::float16_t and
# std::bfloat16_t. Every string is parsed as w * 10^q with w < 10^19, so each
# constant has a condition on (w, q) that it must satisfy.
#
# References:
# Daniel Lemire, Number Parsing at a Gigabyte per Second,
# Software: Practice and Experience 51 (8), 2021 https://arxiv.org/abs/2101.11408
# Noble Mushtak and Daniel Lemire, Fast Number Parsing Without Fallback,
# Software: Practice and Experience 53 (6), 2023 https://arxiv.org/abs/2212.06644
#
# Usage: python3 script/format_parameters.py [--types float16,bfloat16]
# [--sweep-width 3] [--jobs N]
#
import argparse
import multiprocessing
import os
import re
import sys
HEADER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..",
"include", "fast_float", "float_common.h")
MASK64 = (1 << 64) - 1
MAX_W = 10**19
CXX_TYPES = {"double": "double", "float": "float",
"float16": "std::float16_t", "bfloat16": "std::bfloat16_t"}
class Format:
def __init__(self, name):
self.name = name
cxx = CXX_TYPES[name]
with open(HEADER) as f:
text = f.read()
pattern = r"binary_format<" + re.escape(cxx) + r">::(\w+)\(\)\s*\{(.*?)\n\}"
for fn, body in re.findall(pattern, text, re.S):
returns = re.findall(r"return\s+([^;]+);", body)
# the last return is the FLT_EVAL_METHOD 0/1 branch
if returns and re.fullmatch(r"-?(0x[0-9A-Fa-f]+|\d+)", returns[-1].strip()):
setattr(self, fn, int(returns[-1], 0))
tables = re.search(r"binary_format_lookup_tables<" + re.escape(cxx) +
r", U>\s*\{(.*?)\n\};", text, re.S).group(1)
powers = re.search(r"powers_of_ten\[\]\s*=\s*\{(.*?)\}", tables, re.S).group(1)
self.powers_of_ten = [float(x) for x in
re.sub(r"(bf16|f16|f)\b", "", powers).split(",") if x.strip()]
mant = re.search(r"max_mantissa\[\]\s*=\s*\{(.*?)\}", tables, re.S).group(1)
mant = re.sub(r"//[^\n]*", "", mant).replace("/", "//").replace("\n", " ")
self.max_mantissa = [eval(x.strip(), {"constant_55555": 5**5})
for x in mant.split(",") if x.strip()]
self.p = self.mantissa_explicit_bits
self.bias = self.p - self.minimum_exponent
self.denorm_min_exp = 1 - self.bias # smallest subnormal is 2^denorm_min_exp
self.emin = self.minimum_exponent + 1 # leading bit of the smallest normal
self.emax = self.infinite_power - 1 + self.minimum_exponent
def grid(self):
"""Every finite value and every midpoint as (num, exp2, kind) with
kind in value, tie_down (even lower neighbour), tie_up, overflow."""
out = []
two_p = 1 << self.p
d = self.denorm_min_exp
for m in range(1, two_p):
out.append((m, d, "value"))
for m in range(two_p):
out.append((2 * m + 1, d - 1, "tie_down" if m % 2 == 0 else "tie_up"))
for e in range(self.emin, self.emax + 1):
for f in range(two_p):
out.append((two_p + f, e - self.p, "value"))
kind = "tie_down" if f % 2 == 0 else "tie_up"
if e == self.emax and f == two_p - 1:
kind = "overflow"
out.append((2 * (two_p + f) + 1, e - self.p - 1, kind))
return out
def ratio(num, exp2):
return (num << exp2, 1) if exp2 >= 0 else (num, 1 << -exp2)
def ratio10(w, q):
return (w * 10**q, 1) if q >= 0 else (w, 10**-q)
def le(n1, d1, n2, d2):
return n1 * d2 <= n2 * d1
def floor_log2(n, d):
e = n.bit_length() - d.bit_length()
if (d << e if e >= 0 else d) > (n if e >= 0 else n << -e):
e -= 1
return e
def round_ties_even(n, d):
m, r = divmod(n, d)
if 2 * r > d or (2 * r == d and m & 1):
m += 1
return m
def reference_bits(fmt, n, d):
"""Correctly rounded bit pattern of n/d >= 0."""
if n == 0:
return 0
e = floor_log2(n, d)
s = fmt.denorm_min_exp if e < fmt.emin else e - fmt.p
m = round_ties_even(n, d << s) if s >= 0 else round_ties_even(n << -s, d)
if e < fmt.emin:
return m
if m == 2 << fmt.p:
m >>= 1
e += 1
if e > fmt.emax:
return fmt.infinite_power << fmt.p
return ((e - fmt.minimum_exponent) << fmt.p) | (m - (1 << fmt.p))
def ceil_log2(x):
z = 0
while (1 << z) < x:
z += 1
return z
def table_entry(q):
"""T[q] as stored in fast_table.h (see table_generation.py)."""
if q < 0:
power5 = 5**-q
z = ceil_log2(power5)
b = z + 127 if q >= -27 else 2 * z + 128
c = 2**b // power5 + 1
while c >= 1 << 128:
c //= 2
return c
power5 = 5**q
while power5 < 1 << 127:
power5 *= 2
while power5 >= 1 << 128:
power5 //= 2
return power5
TABLE = {q: table_entry(q) for q in range(-342, 309)}
def compute_product_approximation(q, w, bit_precision):
t = TABLE[q]
first = w * (t >> 64)
high, low = first >> 64, first & MASK64
if (high & (MASK64 >> bit_precision)) == MASK64 >> bit_precision:
second_high = (w * (t & MASK64)) >> 64
low = (low + second_high) & MASK64
if second_high > low:
high += 1
return high, low
def compute_float(fmt, q, w, subnormal_tie_test=True):
"""Model of compute_float in decimal_to_binary.h. Returns
(mantissa, power2, truncated mantissa, k) with truncated == floor(w 10^q 2^k)."""
if w == 0 or q < fmt.smallest_power_of_ten:
return 0, 0, None, None
if q > fmt.largest_power_of_ten:
return 0, fmt.infinite_power, None, None
lz = 64 - w.bit_length()
w <<= lz
high, low = compute_product_approximation(q, w, fmt.p + 3)
upperbit = high >> 63
shift = upperbit + 64 - fmt.p - 3
mantissa = high >> shift
power2 = ((152170 + 65536) * q >> 16) + 63 + upperbit - lz - fmt.minimum_exponent
tie_q = fmt.min_exponent_round_to_even <= q <= fmt.max_exponent_round_to_even
if power2 <= 0:
if -power2 + 1 >= 64:
return 0, 0, None, None
extra = -power2 + 1
mantissa >>= extra
truncated, k = mantissa, fmt.bias
if (subnormal_tie_test and low <= 1 and tie_q and (mantissa & 3) == 1
and ((mantissa << extra) << shift) == high):
mantissa &= ~1
mantissa += mantissa & 1
mantissa >>= 1
return mantissa, int(mantissa >= 1 << fmt.p), truncated, k
truncated, k = mantissa, 1 + fmt.bias - power2
if low <= 1 and tie_q and (mantissa & 3) == 1 and (mantissa << shift) == high:
mantissa &= ~1
mantissa += mantissa & 1
mantissa >>= 1
if mantissa >= 2 << fmt.p:
mantissa = 1 << fmt.p
power2 += 1
mantissa &= ~(1 << fmt.p)
if power2 >= fmt.infinite_power:
return 0, fmt.infinite_power, truncated, k
return mantissa, power2, truncated, k
def parsed_bits(fmt, w, q, subnormal_tie_test=True):
"""from_chars on the digits w * 10^q. Clinger's fast path is one correctly
rounded operation on exact operands, so it equals the reference."""
if fmt.min_exponent_fast_path <= q <= fmt.max_exponent_fast_path and w <= 2 << fmt.p:
return reference_bits(fmt, *ratio10(w, q)), None, None
mantissa, power2, truncated, k = compute_float(fmt, q, w, subnormal_tie_test)
return mantissa | (power2 << fmt.p), truncated, k
failures = 0
def check(ok, text):
global failures
print((" ok " if ok else " FAIL ") + text)
failures += not ok
def note(text):
print(" note " + text)
def continued_fraction(numer, denom):
cf = []
while denom:
quot, rem = divmod(numer, denom)
cf.append(quot)
numer, denom = denom, rem
return cf
def convergents(cf):
p2, q2, p1, q1 = 0, 1, 1, 0
for a in cf:
p1, p2, q1, q2 = a * p1 + p2, p1, a * q1 + q2, q1
yield p1, q1
def check_constants(fmt):
half_denorm = ratio(1, fmt.denorm_min_exp - 1)
q = fmt.smallest_power_of_ten
check(le(*ratio10(MAX_W - 1, q - 1), *half_denorm),
"smallest_power_of_ten=%d: q < %d rounds to zero" % (q, q))
check(not le(*ratio10(MAX_W - 1, q), *half_denorm),
"smallest_power_of_ten is tight")
threshold = ratio((4 << fmt.p) - 1, fmt.emax - fmt.p - 1) # max + ulp/2
q = fmt.largest_power_of_ten
check(le(*threshold, *ratio10(1, q + 1)),
"largest_power_of_ten=%d: q > %d overflows" % (q, q))
check(not le(*threshold, *ratio10(1, q)), "largest_power_of_ten is tight")
two_p1 = 2 << fmt.p
q = fmt.max_exponent_fast_path
check(5**q <= two_p1 < 5 ** (q + 1),
"max_exponent_fast_path=%d: 10^q exact in T iff q <= %d" % (q, q))
check(fmt.powers_of_ten[: q + 1] == [10.0**i for i in range(q + 1)], "powers_of_ten")
check(fmt.max_mantissa[: q + 1] == [two_p1 // 5**i for i in range(q + 1)],
"max_mantissa[q] == floor(2^%d / 5^q)" % (fmt.p + 1))
overflow = not le(two_p1 * 10**q, 1, *ratio(two_p1 - 1, fmt.emax - fmt.p))
note("fast path product 2^%d * 10^%d %s the largest finite value"
% (fmt.p + 1, q, "exceeds" if overflow else "is below"))
k = -fmt.min_exponent_fast_path
check(0 <= k <= fmt.max_exponent_fast_path, "min_exponent_fast_path=%d" % -k)
if fmt.p <= 10:
# double rounding: w / 10^k rounded to `wide` bits, then to T
for wide in (24, 53, 64):
bad = 0
for kk in range(1, fmt.max_exponent_fast_path + 1):
for w in range(1, two_p1 + 1):
e = floor_log2(w, 10**kk)
s = e - wide + 1
m = round_ties_even(w, 10**kk << s) if s >= 0 else round_ties_even(w << -s, 10**kk)
if reference_bits(fmt, *ratio(m, s)) != reference_bits(fmt, w, 10**kk):
bad += 1
check(bad == 0, "w / 10^k, k <= %d, via %d-bit intermediate never double-rounds"
% (fmt.max_exponent_fast_path, wide))
def check_ties(fmt):
"""A tie is m * 2^e with m odd. It is w * 10^q, w < 10^19, iff q <= e and
w = m 5^-q 2^(e-q) (q <= 0) or 5^q | m (q > 0). Ties whose lower neighbour
is even (m == 1 mod 4) must fall in the round-to-even range."""
p = fmt.p
lo, hi = fmt.min_exponent_round_to_even, fmt.max_exponent_round_to_even
q = 0
while ((1 << (p + 1)) + 1) * 5 ** (q + 1) < MAX_W:
q += 1
normal_lo = -q
q = 0
while 5 ** (q + 1) < 1 << (p + 2):
q += 1
while True:
c = 1
while 5**q * c <= 1 << (p + 1):
c += 4
if 5**q * c < 1 << (p + 2) and q <= fmt.emax - p - 1:
break
q -= 1
normal_hi = q
e = fmt.denorm_min_exp - 1
sub = []
q = e
while 5**-q << (e - q) < MAX_W:
sub.append(q)
q -= 1
check(lo <= normal_lo and hi >= normal_hi,
"round_to_even range [%d, %d] covers normal ties [%d, %d]" % (lo, hi, normal_lo, normal_hi))
if sub:
check(lo <= min(sub), "and the subnormal ties at q in [%d, %d]" % (min(sub), max(sub)))
else:
note("no subnormal tie has fewer than 20 digits")
# subnormal_ties_possible() in float_common.h: an upper bound on the
# largest q that can give a subnormal, compared with the tie range
q_sub = ((fmt.minimum_exponent + 1) * 1233) >> 12
check(10**q_sub < 2**fmt.emin < 10 ** (q_sub + 2), "q <= %d bounds the subnormal exponents" % q_sub)
check((lo <= q_sub) == bool(sub), "subnormal_ties_possible() == %s" % bool(sub))
longest = len(str(((1 << (p + 2)) - 1) * 5**fmt.bias))
check(fmt.max_digits >= longest, "max_digits=%d >= %d, the longest tie" % (fmt.max_digits, longest))
def check_table(fmt):
"""Facts behind the tie test (product.low <= 1 and dropped bits zero)."""
q_lo, q_hi = max(fmt.smallest_power_of_ten, -27), -1
check(fmt.min_exponent_round_to_even >= -27, "tie test stays where 5^-q < 2^64")
check(all(TABLE[q] & MASK64 for q in range(q_lo, q_hi + 1)),
"T[q] low word nonzero for %d <= q <= %d" % (q_lo, q_hi))
fake = False
for q in range(q_lo, q_hi + 1):
t_hi = TABLE[q] >> 64
v2 = (t_hi & -t_hi).bit_length() - 1
z = ceil_log2(5**-q)
for shift in (61 - fmt.p, 62 - fmt.p):
# skipped second product: w * t_hi == 0 or 1 mod 2^(shift+64)
if shift + 64 - v2 <= 63:
fake = True
if v2 == 0 and (1 << 63) <= pow(t_hi, -1, 1 << (shift + 64)) < 1 << 64:
fake = True
# computed second product: a non-tie within 2^-62 of a rational
# with denominator 2^(z-1-shift) is impossible
if z - 1 - shift > 62:
fake = True
check(not fake, "no false tie for %d <= q <= %d" % (q_lo, q_hi))
bad = False
for q in range(fmt.smallest_power_of_ten, fmt.largest_power_of_ten + 1):
for _, w in convergents(continued_fraction(TABLE[q], 2**137)):
if w >= 2**64:
break
if (TABLE[q] * w) % 2**137 > 2**137 - 2**64:
bad = True
check(not bad, "Mushtak-Lemire condition for %d <= q <= %d"
% (fmt.smallest_power_of_ten, fmt.largest_power_of_ten))
def sweep_q(args):
fmt, grid, q, width = args
cands = set()
for num, exp2, _ in grid:
n, d = ratio(num, exp2)
if q >= 0:
d *= 10**q
else:
n *= 10**-q
cands.update(w for w in range(n // d - width, n // d + width + 2) if 1 <= w < MAX_W)
wrong, wrong_without, bad_trunc = [], 0, 0
for w in cands:
n, d = ratio10(w, q)
want = reference_bits(fmt, n, d)
got, truncated, k = parsed_bits(fmt, w, q)
if got != want:
wrong.append((w, q, got, want))
if truncated is not None:
exact = (n << k) // d if k >= 0 else n // (d << -k)
bad_trunc += truncated != exact
if parsed_bits(fmt, w, q, False)[0] != want:
wrong_without += 1
return len(cands), wrong, wrong_without, bad_trunc
def sweep(fmt, width, jobs):
"""Compare the model with the reference on every w within `width` of
G / 10^q for every value, tie and the overflow threshold G."""
grid = fmt.grid()
qs = range(fmt.smallest_power_of_ten - 1, fmt.largest_power_of_ten + 2)
tasks = [(fmt, grid, q, width) for q in qs]
with multiprocessing.Pool(jobs) as pool:
results = pool.map(sweep_q, tasks)
wrong = [x for r in results for x in r[1]]
note("swept %d pairs (w, q)" % sum(r[0] for r in results))
check(sum(r[3] for r in results) == 0, "truncated mantissa is exact")
check(not wrong, "compute_float matches the reference")
for w, q, got, want in sorted(wrong)[:20]:
print(" w=%d q=%d got 0x%04x want 0x%04x" % (w, q, got, want))
note("without the subnormal tie test: %d wrong" % sum(r[2] for r in results))
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--types", default="double,float,float16,bfloat16")
ap.add_argument("--sweep-width", type=int, default=3)
ap.add_argument("--jobs", type=int, default=os.cpu_count() or 1)
args = ap.parse_args()
for name in args.types.split(","):
fmt = Format(name.strip())
print("== %s" % fmt.name)
check_constants(fmt)
check_table(fmt)
check_ties(fmt)
if fmt.p <= 10 and args.sweep_width > 0:
sweep(fmt, args.sweep_width, args.jobs)
print("%d failure(s)" % failures)
return 1 if failures else 0
if __name__ == "__main__":
sys.exit(main())

2
tests/CMakeLists.txt
View File

@ -95,6 +95,8 @@ endif()
if (FASTFLOAT_FIXEDWIDTH_TESTS)
fast_float_add_cpp_test(fixedwidthtest)
target_compile_features(fixedwidthtest PUBLIC cxx_std_23)
fast_float_add_cpp_test(exhaustive16_midpoint)
target_compile_features(exhaustive16_midpoint PUBLIC cxx_std_23)
endif()
option(FASTFLOAT_EXHAUSTIVE "Exhaustive tests" OFF)

30
tests/basictest.cpp
View File

@ -1828,6 +1828,20 @@ TEST_CASE("float16.inf") {
std::errc::result_out_of_range);
verify("3.5028234666e38", std::numeric_limits<std::float16_t>::infinity(),
std::errc::result_out_of_range);
// overflow in the fast path
verify("656e2", std::numeric_limits<std::float16_t>::infinity(),
std::errc::result_out_of_range);
verify("-656e2", -std::numeric_limits<std::float16_t>::infinity(),
std::errc::result_out_of_range);
verify("7e4", std::numeric_limits<std::float16_t>::infinity(),
std::errc::result_out_of_range);
verify("2048e4", std::numeric_limits<std::float16_t>::infinity(),
std::errc::result_out_of_range);
verify("655e2", 0x1.ffcp+15f16); // 65500 rounds to max
// max + ulp/2 rounds to even
verify("6552e1", std::numeric_limits<std::float16_t>::infinity(),
std::errc::result_out_of_range);
verify("65519", 0x1.ffcp+15f16);
}
TEST_CASE("float16.general") {
@ -1848,6 +1862,22 @@ TEST_CASE("float16.general") {
verify("-0.000000059604644775390625", -0x1p-24f16);
verify("-5.9604644775390625e-8", -0x1p-24f16);
// subnormal ties round to even
verify("2.98023223876953125e-8", 0.0f16, std::errc::result_out_of_range);
verify("0.0000000298023223876953125", 0.0f16, std::errc::result_out_of_range);
verify("-2.98023223876953125e-8", -0.0f16, std::errc::result_out_of_range);
verify("8.94069671630859375e-8", 0x1p-23f16);
verify("1.490116119384765625e-7", 0x1p-23f16);
verify("9.834766387939453125e-7", 0x1p-20f16);
// just above or below a tie
verify("2.980232238769531251e-8", 0x1p-24f16);
verify("2.980232238769531249e-8", 0.0f16, std::errc::result_out_of_range);
verify("1.4901161193847656251e-7", 0x1.8p-23f16);
verify("1.4901161193847656249e-7", 0x1p-23f16);
// more than 19 digits
verify("2.9802322387695312500e-8", 0.0f16, std::errc::result_out_of_range);
verify("1.4901161193847656250000e-7", 0x1p-23f16);
verify("-1e-999", -0.0f16, std::errc::result_out_of_range);
verify("6.0975551605224609375", 0x1.864p+2f16);
verify_runtime(append_zeros("6.0975551605224609375", 655), 0x1.864p+2f16);

163
tests/exhaustive16_midpoint.cpp Normal file
View File

@ -0,0 +1,163 @@
// For std::float16_t and std::bfloat16_t, parse the exact decimal expansion
// of every finite value and of every midpoint between adjacent values, plus
// strings just above and below each midpoint and midpoints padded past 19
// digits. The expected bits follow from the bit pattern that generated the
// string, so no C library reference is needed.
#include "fast_float/fast_float.h"
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <stdfloat>
#include <string>
#include <system_error>
namespace {
// Decimal digit strings, most significant digit first, no leading zeros.
std::string times(std::string const &digits, unsigned factor) {
std::string out;
unsigned carry = 0;
for (size_t i = digits.size(); i-- > 0;) {
unsigned v = static_cast<unsigned>(digits[i] - '0') * factor + carry;
out.push_back(static_cast<char>('0' + v % 10));
carry = v / 10;
}
while (carry != 0) {
out.push_back(static_cast<char>('0' + carry % 10));
carry /= 10;
}
return std::string(out.rbegin(), out.rend());
}
std::string minus_one(std::string digits) {
size_t i = digits.size();
while (i-- > 0) {
if (digits[i] != '0') {
digits[i] = static_cast<char>(digits[i] - 1);
break;
}
digits[i] = '9';
}
return digits;
}
// num * 2^exp2 written as "<digits>e<exponent>" with an exact expansion.
struct Decimal {
std::string digits;
int exponent;
std::string str() const { return digits + "e" + std::to_string(exponent); }
Decimal above() const { return {digits + "1", exponent - 1}; }
Decimal below() const { return {minus_one(digits) + "9", exponent - 1}; }
Decimal padded(size_t count) const {
return {digits + std::string(count, '0'),
exponent - static_cast<int>(count)};
}
};
Decimal exact_decimal(uint64_t num, int exp2) {
std::string digits = std::to_string(num);
if (exp2 >= 0) {
for (int i = 0; i < exp2; i++) {
digits = times(digits, 2);
}
return {digits, 0};
}
for (int i = 0; i < -exp2; i++) {
digits = times(digits, 5);
}
return {digits, exp2};
}
template <typename T> struct Layout;
template <> struct Layout<std::float16_t> {
static constexpr char const *name = "float16";
static constexpr int p = 10;
static constexpr int bias = 25; // value = mantissa * 2^(field - bias)
static constexpr uint16_t infinity = 0x7C00;
};
template <> struct Layout<std::bfloat16_t> {
static constexpr char const *name = "bfloat16";
static constexpr int p = 7;
static constexpr int bias = 134;
static constexpr uint16_t infinity = 0x7F80;
};
struct Failure {
long count = 0;
long checked = 0;
};
template <typename T>
void expect(Failure &f, std::string const &s, uint16_t want) {
T value{};
auto r = fast_float::from_chars(s.data(), s.data() + s.size(), value);
uint16_t got = 0;
std::memcpy(&got, &value, sizeof(got));
bool zero_or_inf = want == 0 || want == Layout<T>::infinity;
std::errc want_ec =
zero_or_inf ? std::errc::result_out_of_range : std::errc();
f.checked++;
if (got != want || r.ec != want_ec || r.ptr != s.data() + s.size()) {
f.count++;
if (f.count <= 20) {
std::printf(
"%s: \"%s\" parsed to 0x%04x (ec %d), expected 0x%04x (ec %d)\n",
Layout<T>::name, s.c_str(), got, static_cast<int>(r.ec), want,
static_cast<int>(want_ec));
}
}
}
template <typename T> Failure sweep() {
Failure f;
constexpr int p = Layout<T>::p;
constexpr uint16_t mantissa_mask = static_cast<uint16_t>((1u << p) - 1);
for (uint32_t b = 0; b < Layout<T>::infinity; b++) {
uint16_t bits = static_cast<uint16_t>(b);
int field = bits >> p;
uint64_t num = bits & mantissa_mask;
int exp2 = (field == 0 ? 1 : field) - Layout<T>::bias;
if (field != 0) {
num |= uint64_t(1) << p;
}
if (num != 0) {
Decimal d = exact_decimal(num, exp2);
expect<T>(f, d.str(), bits);
expect<T>(f, d.padded(21).str(), bits);
}
// The tie between bits and bits + 1 is (2 num + 1) * 2^(exp2 - 1); the
// last one (bits + 1 == infinity) is the overflow threshold.
Decimal tie = exact_decimal(2 * num + 1, exp2 - 1);
uint16_t lower = bits;
uint16_t upper = static_cast<uint16_t>(bits + 1);
uint16_t even = (bits & 1) ? upper : lower;
expect<T>(f, tie.str(), even);
expect<T>(f, tie.padded(21).str(), even);
expect<T>(f, tie.above().str(), upper);
expect<T>(f, tie.below().str(), lower);
}
return f;
}
} // namespace
int main() {
Failure f16 = sweep<std::float16_t>();
std::printf("float16: %ld strings, %ld failures\n", f16.checked, f16.count);
Failure bf16 = sweep<std::bfloat16_t>();
std::printf("bfloat16: %ld strings, %ld failures\n", bf16.checked,
bf16.count);
if (f16.count != 0 || bf16.count != 0) {
return EXIT_FAILURE;
}
std::printf("all ok\n");
return EXIT_SUCCESS;
}