From b2e41e8b8807cf200e54a1a8613515a52501c008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 21 Sep 2026 19:45:12 +0000 Subject: [PATCH] Keep chars_format::javascript out of the common hot path 8.3.0 added the ECMAScript leading-zero rule as a runtime test on fmt inside parse_number_string, so unlike basic_json_fmt its error returns and Annex B octal scan land in the parser body that every from_chars instantiation force-inlines. from_chars_caller::call grew from 935 to 1031 instructions and other formats lost about 3% on short inputs. Make the rule a compile-time parameter of the parser body, selected once at the entry point, and take chars_format::javascript out to a cold conversion before anything else is live so the branch is a tail call. parse_number_string keeps its signature and behaviour. benchmarks/benchmark.cpp, Intel Xeon 6975P-C, clang 22.1.8, cycles per float, best of 7: canada.txt mesh.txt 8.2.10 47.03 24.78 8.3.0 47.34 26.08 this 46.28 24.86 --- include/fast_float/ascii_number.h | 43 ++++++++++++++++++++++--- include/fast_float/float_common.h | 11 +++++++ include/fast_float/parse_number.h | 52 ++++++++++++++++++++++++++++--- 3 files changed, 97 insertions(+), 9 deletions(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index 52f1e84..e814f12 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -339,10 +339,11 @@ report_parse_error(UC const *p, parse_error error) { // spans (read only by the rare digit_comp slow path) are not materialized, // which keeps the fat parsed_number_string_t off the hot path. The caller // re-parses with store_spans=true if the slow path is actually reached. -template +template fastfloat_really_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t -parse_number_string(UC const *p, UC const *pend, parse_options_t options, - bool store_spans = true) noexcept { +parse_number_string_impl(UC const *p, UC const *pend, + parse_options_t options, + bool store_spans) noexcept { chars_format const fmt = detail::adjust_for_feature_macros(options.format); UC const decimal_point = options.decimal_point; @@ -424,7 +425,7 @@ parse_number_string(UC const *p, UC const *pend, parse_options_t options, parse_error::leading_zeros_in_integer_part); } } - else if (uint64_t(fmt & detail::basic_javascript_fmt)) { + else FASTFLOAT_IF_CONSTEXPR17(basic_javascript_fmt) { // ECMAScript DecimalIntegerLiteral is "0" or a non-zero digit followed by // digits: no leading zeros. Unlike JSON, the integer part may be empty // (".5"); the no_digits_in_mantissa check below still rejects ".". @@ -587,6 +588,40 @@ parse_number_string(UC const *p, UC const *pend, parse_options_t options, return answer; } +// Cold instantiation of the parser: the ECMAScript integer-part rule costs two +// error returns plus the Annex B octal scan, and parse_number_string_impl is +// force-inlined, so an inlined javascript body would enlarge the frame of every +// caller that never asks for it. Out of line, it costs those callers nothing. +template +fastfloat_never_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t +parse_number_string_javascript(UC const *p, UC const *pend, + parse_options_t options, + bool store_spans) noexcept { + return parse_number_string_impl(p, pend, options, + store_spans); +} + +// Public entry point, behaviour unchanged: chars_format::javascript is still +// honoured, it is just selected here once instead of being re-tested inside the +// parser loop. Callers that have already ruled the format out (see +// from_chars_float_advanced) should call parse_number_string_impl directly so +// that not even this test reaches their hot path. +template +fastfloat_really_inline FASTFLOAT_CONSTEXPR20 parsed_number_string_t +parse_number_string(UC const *p, UC const *pend, parse_options_t options, + bool store_spans = true) noexcept { + // JSON and JavaScript are mutually exclusive, so only the non-JSON + // instantiation has to look at the flag. + FASTFLOAT_IF_CONSTEXPR17(!basic_json_fmt) { + if fastfloat_unlikely (uint64_t(options.format & + detail::basic_javascript_fmt)) { + return parse_number_string_javascript(p, pend, options, store_spans); + } + } + return parse_number_string_impl(p, pend, options, + store_spans); +} + template fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t parse_int_string(UC const *p, UC const *pend, T &value, diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index 87c72b0..3769972 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -217,6 +217,17 @@ using parse_options = parse_options_t; #define fastfloat_really_inline inline __attribute__((always_inline)) #endif +// Opposite of fastfloat_really_inline. Used for a rare, format-specific variant +// of a force-inlined function, so that its body does not land in the frame of +// every caller that will never execute it. +#ifdef FASTFLOAT_VISUAL_STUDIO +#define fastfloat_never_inline __declspec(noinline) +#elif defined(__GNUC__) || defined(__clang__) +#define fastfloat_never_inline __attribute__((noinline)) +#else +#define fastfloat_never_inline +#endif + // Branch-probability hint marking the rare slow-path branches as cold, so the // optimizer keeps the out-of-line slow-path re-parse off the hot path (and does // not duplicate the force-inlined hot scanner into the caller, which bloated diff --git a/include/fast_float/parse_number.h b/include/fast_float/parse_number.h index 58cdafd..14d077b 100644 --- a/include/fast_float/parse_number.h +++ b/include/fast_float/parse_number.h @@ -306,8 +306,41 @@ FASTFLOAT_CONSTEXPR20 from_chars_result_t parse_number_slow_path(UC const *first, UC const *last, T &value, parse_options_t options, bool bjf) noexcept { parsed_number_string_t pns = - bjf ? parse_number_string(first, last, options, true) - : parse_number_string(first, last, options, true); + bjf ? parse_number_string_impl(first, last, options, + true) + : parse_number_string_impl(first, last, options, + true); + return from_chars_advanced(pns, value); +} + +// Cold: the whole chars_format::javascript conversion. Kept out of line and out +// of from_chars_float_advanced so the common frame holds neither the javascript +// parser body nor a call that would force the parsed number string onto the +// stack. Mirrors the main path, only with the javascript parser. +template +fastfloat_never_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_float_javascript(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + chars_format const fmt = detail::adjust_for_feature_macros(options.format); + if (uint64_t(fmt & chars_format::skip_white_space)) { + while ((first != last) && fast_float::is_space(*first)) { + first++; + } + } + from_chars_result_t answer; + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + if (first == last) { + return answer; + } + parsed_number_string_t pns = + parse_number_string_javascript(first, last, options, true); + if (!pns.valid) { + if (uint64_t(fmt & chars_format::no_infnan)) { + return answer; + } + return detail::parse_infnan(first, last, value, fmt); + } return from_chars_advanced(pns, value); } @@ -323,6 +356,13 @@ from_chars_float_advanced(UC const *first, UC const *last, T &value, chars_format const fmt = detail::adjust_for_feature_macros(options.format); + // Leave for the javascript conversion before anything else is live, so this + // compiles to a tail call and the common path keeps its registers. The cold + // function repeats the leading-whitespace and empty-input handling below. + if fastfloat_unlikely (uint64_t(fmt & detail::basic_javascript_fmt)) { + return from_chars_float_javascript(first, last, value, options); + } + from_chars_result_t answer; if (uint64_t(fmt & chars_format::skip_white_space)) { while ((first != last) && fast_float::is_space(*first)) { @@ -339,10 +379,12 @@ from_chars_float_advanced(UC const *first, UC const *last, T &value, // Fast path: parse WITHOUT materializing the integer/fraction spans (read // only by the rare slow paths). Skipping their stores keeps the fat // parsed_number_string_t off the hot path. store_spans is a runtime argument, - // so this reuses the single parse_number_string instantiation. + // so this reuses the single parse_number_string_impl instantiation. parsed_number_string_t pns = - bjf ? parse_number_string(first, last, options, false) - : parse_number_string(first, last, options, false); + bjf ? parse_number_string_impl(first, last, options, + false) + : parse_number_string_impl(first, last, options, + false); if (!pns.valid) { if (uint64_t(fmt & chars_format::no_infnan)) { answer.ec = std::errc::invalid_argument;