diff --git a/benchmarks/benchmark.cpp b/benchmarks/benchmark.cpp index 86d7616..94cad23 100644 --- a/benchmarks/benchmark.cpp +++ b/benchmarks/benchmark.cpp @@ -1,266 +1,270 @@ - -#define FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN -#define FASTFLOAT_ONLY_ROUNDS_TO_NEAREST_SUPPORTED -#define FASTFLOAT_ISNOT_CHECKED_BOUNDS - -#if defined(__linux__) || (__APPLE__ && __aarch64__) -#define USING_COUNTERS -#endif -#include "counters/event_counter.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "fast_float/fast_float.h" - -template -Value findmax_fastfloat(std::vector> &s) { - Value answer = 0; - Value x = 0; - for (auto &st : s) { - auto [p, ec] = fast_float::from_chars(st.data(), st.data() + st.size(), x); - - if (p == st.data()) { - throw std::runtime_error("bug in findmax_fastfloat"); - } - answer = answer > x ? answer : x; - } - return answer; -} - -#ifdef USING_COUNTERS - -counters::event_collector collector{}; - -template -std::vector -time_it_ns(std::vector> &lines, T const &function, - uint32_t repeat) { - std::vector aggregate; - bool printed_bug = false; - for (uint32_t i = 0; i != repeat; ++i) { - collector.start(); - auto const ts = function(lines); - aggregate.push_back(collector.end()); - - if (ts == 0 && !printed_bug) { - printf("bug\n"); - printed_bug = true; - } - } - return aggregate; -} - -void pretty_print(uint64_t volume, size_t number_of_floats, std::string name, - std::vector events) { - double volumeMB = volume / (1024. * 1024.); - double average_ns{0}; - double min_ns{DBL_MAX}; - double cycles_min{DBL_MAX}; - double instructions_min{DBL_MAX}; - double cycles_avg{0}; - double instructions_avg{0}; - double branches_min{0}; - double branches_avg{0}; - double branch_misses_min{0}; - double branch_misses_avg{0}; - for (counters::event_count e : events) { - double ns = e.elapsed_ns(); - average_ns += ns; - min_ns = min_ns < ns ? min_ns : ns; - - double cycles = e.cycles(); - cycles_avg += cycles; - cycles_min = cycles_min < cycles ? cycles_min : cycles; - - double instructions = e.instructions(); - instructions_avg += instructions; - instructions_min = - instructions_min < instructions ? instructions_min : instructions; - - double branches = e.branches(); - branches_avg += branches; - branches_min = branches_min < branches ? branches_min : branches; - - double branch_misses = e.branch_misses(); - branch_misses_avg += branch_misses; - branch_misses_min = - branch_misses_min < branch_misses ? branch_misses_min : branch_misses; - } - cycles_avg /= events.size(); - instructions_avg /= events.size(); - average_ns /= events.size(); - branches_avg /= events.size(); - printf("%-40s: %8.2f MB/s (+/- %.1f %%) ", name.data(), - volumeMB * 1000000000 / min_ns, - (average_ns - min_ns) * 100.0 / average_ns); - printf("%8.2f Mfloat/s ", number_of_floats * 1000 / min_ns); - if (instructions_min > 0) { - printf(" %8.2f i/B %8.2f i/f (+/- %.1f %%) ", instructions_min / volume, - instructions_min / number_of_floats, - (instructions_avg - instructions_min) * 100.0 / instructions_avg); - - printf(" %8.2f c/B %8.2f c/f (+/- %.1f %%) ", cycles_min / volume, - cycles_min / number_of_floats, - (cycles_avg - cycles_min) * 100.0 / cycles_avg); - printf(" %8.2f i/c ", instructions_min / cycles_min); - printf(" %8.2f b/f ", branches_avg / number_of_floats); - printf(" %8.2f bm/f ", branch_misses_avg / number_of_floats); - printf(" %8.2f GHz ", cycles_min / min_ns); - } - printf("\n"); -} -#else -template -std::pair -time_it_ns(std::vector> &lines, T const &function, - size_t repeat) { - std::chrono::high_resolution_clock::time_point t1, t2; - double average = 0; - double min_value = DBL_MAX; - bool printed_bug = false; - for (size_t i = 0; i != repeat; ++i) { - t1 = std::chrono::high_resolution_clock::now(); - auto const ts = function(lines); - t2 = std::chrono::high_resolution_clock::now(); - - double const dif = static_cast( - std::chrono::duration_cast(t2 - t1).count()); - average += dif; - min_value = min_value < dif ? min_value : dif; - - if (ts == 0 && !printed_bug) { - printf("bug\n"); - printed_bug = true; - } - } - average /= repeat; - return std::make_pair(min_value, average); -} - -void pretty_print(uint64_t volume, size_t number_of_floats, - std::string const &name, std::pair result) { - double volumeMB = volume / (1024. * 1024.); - printf("%-40s: %8.2f MB/s (+/- %.1f %%) ", name.data(), - volumeMB * 1000000000 / result.first, - (result.second - result.first) * 100.0 / result.second); - printf("%8.2f Mfloat/s ", number_of_floats * 1000 / result.first); - printf(" %8.2f ns/f \n", double(result.first) / number_of_floats); -} -#endif - -// this is okay, all chars are ASCII -inline std::u16string widen(std::string const &line) { - std::u16string u16line; - u16line.resize(line.size()); - for (uint32_t i = 0; i != line.size(); ++i) { - u16line[i] = char16_t(line[i]); - } - return u16line; -} - -std::vector widen(const std::vector &lines) { - std::vector u16lines; - u16lines.reserve(lines.size()); - for (auto const &line : lines) { - u16lines.emplace_back(widen(line)); - } - return u16lines; -} - -void process(std::vector &lines, size_t volume) { - size_t constexpr repeat = 1000; - double volumeMB = volume / (1024. * 1024.); - std::cout << "ASCII volume = " << volumeMB << " MB " << std::endl; - pretty_print(volume, lines.size(), "fastfloat (64)", - time_it_ns(lines, findmax_fastfloat, repeat)); - pretty_print(volume, lines.size(), "fastfloat (32)", - time_it_ns(lines, findmax_fastfloat, repeat)); - - std::vector lines16 = widen(lines); - volume = 2 * volume; - volumeMB = volume / (1024. * 1024.); - std::cout << "UTF-16 volume = " << volumeMB << " MB " << std::endl; - pretty_print( - volume, lines.size(), "fastfloat (64)", - time_it_ns(lines16, findmax_fastfloat, repeat)); - pretty_print(volume, lines.size(), "fastfloat (32)", - time_it_ns(lines16, findmax_fastfloat, repeat)); -} - -void fileload(std::string filename) { - std::ifstream inputfile(filename); - if (!inputfile) { - std::cerr << "can't open " << filename << std::endl; - return; - } - std::cout << "#### " << std::endl; - std::cout << "# reading " << filename << std::endl; - std::cout << "#### " << std::endl; - std::string line; - std::vector lines; - lines.reserve(120000); // let us reserve plenty of memory. - size_t volume = 0; - while (getline(inputfile, line)) { -#ifdef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN - if (line[0] == '-') { - line.erase(0, 1); - } -#endif - volume += line.size(); - lines.emplace_back(line); - } - std::cout << "# read " << lines.size() << " lines " << std::endl; - process(lines, volume); -} - -int main(int argc, char **argv) { -#ifdef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN - std::cout << "# FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN is enabled" - << std::endl; -#endif -#ifdef FASTFLOAT_TABLE_HACK_CHAR_DIGIT_LUT_DISABLED - std::cout << "# FASTFLOAT_TABLE_HACK_CHAR_DIGIT_LUT_DISABLED is enabled" - << std::endl; -#endif -#ifdef FASTFLOAT_ONLY_ROUNDS_TO_NEAREST_SUPPORTED - std::cout << "# FASTFLOAT_ONLY_ROUNDS_TO_NEAREST_SUPPORTED is enabled" - << std::endl; -#endif -#ifdef FASTFLOAT_ISNOT_CHECKED_BOUNDS - std::cout << "# FASTFLOAT_ISNOT_CHECKED_BOUNDS is enabled" << std::endl; -#endif -#ifdef USING_COUNTERS - if (collector.has_events()) { - std::cout << "# Using hardware counters" << std::endl; - } else { -#if defined(__linux__) || (__APPLE__ && __aarch64__) - std::cout << "# Hardware counters not available, try to run in privileged " - "mode (e.g., sudo)." - << std::endl; -#endif - } -#endif - if (argc > 1) { - fileload(argv[1]); - return EXIT_SUCCESS; - } - - fileload(std::string(BENCHMARK_DATA_DIR) + "/canada.txt"); - fileload(std::string(BENCHMARK_DATA_DIR) + "/canada_short.txt"); - fileload(std::string(BENCHMARK_DATA_DIR) + "/mesh.txt"); - return EXIT_SUCCESS; -} + +#define FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN +#define FASTFLOAT_ONLY_ROUNDS_TO_NEAREST_SUPPORTED +#define FASTFLOAT_ISNOT_CHECKED_BOUNDS +// #define FASTFLOAT_SSE_PATCH_REWORK // WIP, only for test! +#ifdef FASTFLOAT_SSE_PATCH_REWORK + #define FASTFLOAT_X86_SIMD 42 +#endif + +#if defined(__linux__) || (__APPLE__ && __aarch64__) +#define USING_COUNTERS +#endif +#include "counters/event_counter.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fast_float/fast_float.h" + +template +Value findmax_fastfloat(std::vector> &s) { + Value answer = 0; + Value x = 0; + for (auto &st : s) { + auto [p, ec] = fast_float::from_chars(st.data(), st.data() + st.size(), x); + + if (p == st.data()) { + throw std::runtime_error("bug in findmax_fastfloat"); + } + answer = answer > x ? answer : x; + } + return answer; +} + +#ifdef USING_COUNTERS + +counters::event_collector collector{}; + +template +std::vector +time_it_ns(std::vector> &lines, T const &function, + uint32_t repeat) { + std::vector aggregate; + bool printed_bug = false; + for (uint32_t i = 0; i != repeat; ++i) { + collector.start(); + auto const ts = function(lines); + aggregate.push_back(collector.end()); + + if (ts == 0 && !printed_bug) { + printf("bug\n"); + printed_bug = true; + } + } + return aggregate; +} + +void pretty_print(uint64_t volume, size_t number_of_floats, std::string name, + std::vector events) { + double volumeMB = volume / (1024. * 1024.); + double average_ns{0}; + double min_ns{DBL_MAX}; + double cycles_min{DBL_MAX}; + double instructions_min{DBL_MAX}; + double cycles_avg{0}; + double instructions_avg{0}; + double branches_min{0}; + double branches_avg{0}; + double branch_misses_min{0}; + double branch_misses_avg{0}; + for (counters::event_count e : events) { + double ns = e.elapsed_ns(); + average_ns += ns; + min_ns = min_ns < ns ? min_ns : ns; + + double cycles = e.cycles(); + cycles_avg += cycles; + cycles_min = cycles_min < cycles ? cycles_min : cycles; + + double instructions = e.instructions(); + instructions_avg += instructions; + instructions_min = + instructions_min < instructions ? instructions_min : instructions; + + double branches = e.branches(); + branches_avg += branches; + branches_min = branches_min < branches ? branches_min : branches; + + double branch_misses = e.branch_misses(); + branch_misses_avg += branch_misses; + branch_misses_min = + branch_misses_min < branch_misses ? branch_misses_min : branch_misses; + } + cycles_avg /= events.size(); + instructions_avg /= events.size(); + average_ns /= events.size(); + branches_avg /= events.size(); + printf("%-40s: %8.2f MB/s (+/- %.1f %%) ", name.data(), + volumeMB * 1000000000 / min_ns, + (average_ns - min_ns) * 100.0 / average_ns); + printf("%8.2f Mfloat/s ", number_of_floats * 1000 / min_ns); + if (instructions_min > 0) { + printf(" %8.2f i/B %8.2f i/f (+/- %.1f %%) ", instructions_min / volume, + instructions_min / number_of_floats, + (instructions_avg - instructions_min) * 100.0 / instructions_avg); + + printf(" %8.2f c/B %8.2f c/f (+/- %.1f %%) ", cycles_min / volume, + cycles_min / number_of_floats, + (cycles_avg - cycles_min) * 100.0 / cycles_avg); + printf(" %8.2f i/c ", instructions_min / cycles_min); + printf(" %8.2f b/f ", branches_avg / number_of_floats); + printf(" %8.2f bm/f ", branch_misses_avg / number_of_floats); + printf(" %8.2f GHz ", cycles_min / min_ns); + } + printf("\n"); +} +#else +template +std::pair +time_it_ns(std::vector> &lines, T const &function, + size_t repeat) { + std::chrono::high_resolution_clock::time_point t1, t2; + double average = 0; + double min_value = DBL_MAX; + bool printed_bug = false; + for (size_t i = 0; i != repeat; ++i) { + t1 = std::chrono::high_resolution_clock::now(); + auto const ts = function(lines); + t2 = std::chrono::high_resolution_clock::now(); + + double const dif = static_cast( + std::chrono::duration_cast(t2 - t1).count()); + average += dif; + min_value = min_value < dif ? min_value : dif; + + if (ts == 0 && !printed_bug) { + printf("bug\n"); + printed_bug = true; + } + } + average /= repeat; + return std::make_pair(min_value, average); +} + +void pretty_print(uint64_t volume, size_t number_of_floats, + std::string const &name, std::pair result) { + double volumeMB = volume / (1024. * 1024.); + printf("%-40s: %8.2f MB/s (+/- %.1f %%) ", name.data(), + volumeMB * 1000000000 / result.first, + (result.second - result.first) * 100.0 / result.second); + printf("%8.2f Mfloat/s ", number_of_floats * 1000 / result.first); + printf(" %8.2f ns/f \n", double(result.first) / number_of_floats); +} +#endif + +// this is okay, all chars are ASCII +inline std::u16string widen(std::string const &line) { + std::u16string u16line; + u16line.resize(line.size()); + for (uint32_t i = 0; i != line.size(); ++i) { + u16line[i] = char16_t(line[i]); + } + return u16line; +} + +std::vector widen(const std::vector &lines) { + std::vector u16lines; + u16lines.reserve(lines.size()); + for (auto const &line : lines) { + u16lines.emplace_back(widen(line)); + } + return u16lines; +} + +void process(std::vector &lines, size_t volume) { + size_t constexpr repeat = 1000; + double volumeMB = volume / (1024. * 1024.); + std::cout << "ASCII volume = " << volumeMB << " MB " << std::endl; + pretty_print(volume, lines.size(), "fastfloat (64)", + time_it_ns(lines, findmax_fastfloat, repeat)); + pretty_print(volume, lines.size(), "fastfloat (32)", + time_it_ns(lines, findmax_fastfloat, repeat)); + + std::vector lines16 = widen(lines); + volume = 2 * volume; + volumeMB = volume / (1024. * 1024.); + std::cout << "UTF-16 volume = " << volumeMB << " MB " << std::endl; + pretty_print( + volume, lines.size(), "fastfloat (64)", + time_it_ns(lines16, findmax_fastfloat, repeat)); + pretty_print(volume, lines.size(), "fastfloat (32)", + time_it_ns(lines16, findmax_fastfloat, repeat)); +} + +void fileload(std::string filename) { + std::ifstream inputfile(filename); + if (!inputfile) { + std::cerr << "can't open " << filename << std::endl; + return; + } + std::cout << "#### " << std::endl; + std::cout << "# reading " << filename << std::endl; + std::cout << "#### " << std::endl; + std::string line; + std::vector lines; + lines.reserve(120000); // let us reserve plenty of memory. + size_t volume = 0; + while (getline(inputfile, line)) { +#ifdef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN + if (line[0] == '-') { + line.erase(0, 1); + } +#endif + volume += line.size(); + lines.emplace_back(line); + } + std::cout << "# read " << lines.size() << " lines " << std::endl; + process(lines, volume); +} + +int main(int argc, char **argv) { +#ifdef FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN + std::cout << "# FASTFLOAT_ONLY_POSITIVE_C_NUMBER_WO_INF_NAN is enabled" + << std::endl; +#endif +#ifdef FASTFLOAT_TABLE_HACK_CHAR_DIGIT_LUT_DISABLED + std::cout << "# FASTFLOAT_TABLE_HACK_CHAR_DIGIT_LUT_DISABLED is enabled" + << std::endl; +#endif +#ifdef FASTFLOAT_ONLY_ROUNDS_TO_NEAREST_SUPPORTED + std::cout << "# FASTFLOAT_ONLY_ROUNDS_TO_NEAREST_SUPPORTED is enabled" + << std::endl; +#endif +#ifdef FASTFLOAT_ISNOT_CHECKED_BOUNDS + std::cout << "# FASTFLOAT_ISNOT_CHECKED_BOUNDS is enabled" << std::endl; +#endif +#ifdef USING_COUNTERS + if (collector.has_events()) { + std::cout << "# Using hardware counters" << std::endl; + } else { +#if defined(__linux__) || (__APPLE__ && __aarch64__) + std::cout << "# Hardware counters not available, try to run in privileged " + "mode (e.g., sudo)." + << std::endl; +#endif + } +#endif + if (argc > 1) { + fileload(argv[1]); + return EXIT_SUCCESS; + } + + fileload(std::string(BENCHMARK_DATA_DIR) + "/canada.txt"); + fileload(std::string(BENCHMARK_DATA_DIR) + "/canada_short.txt"); + fileload(std::string(BENCHMARK_DATA_DIR) + "/mesh.txt"); + return EXIT_SUCCESS; +} diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index 75153af..c7c5cfb 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -10,9 +10,23 @@ #include "float_common.h" -#if defined(FASTFLOAT_SSE2) +#if defined(FASTFLOAT_X86_SIMD) #include -#elif defined(FASTFLOAT_NEON) +#ifdef FASTFLOAT_SSE_PATCH_REWORK +#if FASTFLOAT_X86_SIMD >= 30 +#include +#endif +#if FASTFLOAT_X86_SIMD >= 31 +#include +#endif +#if FASTFLOAT_X86_SIMD >= 41 +#include +#endif +#if FASTFLOAT_X86_SIMD >= 42 +#include +#endif +#endif +#elif defined(FASTFLOAT_ARM_NEON) #include #endif @@ -67,7 +81,7 @@ read_chars_to_unsigned(UC const *chars) noexcept { return val; } -#ifdef FASTFLOAT_SSE2 +#ifdef FASTFLOAT_X86_SIMD fastfloat_really_inline uint64_t simd_read8_to_u64(__m128i const data) { // _mm_packus_epi16 is SSE2+, converts 8×u16 → 8×u8 @@ -90,7 +104,7 @@ fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) { FASTFLOAT_SIMD_RESTORE_WARNINGS } -#elif defined(FASTFLOAT_NEON) +#elif defined(FASTFLOAT_ARM_NEON) fastfloat_really_inline uint64_t simd_read8_to_u64(uint16x8_t const &data) { uint8x8_t utf8_packed = vmovn_u16(data); @@ -104,7 +118,7 @@ fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) { FASTFLOAT_SIMD_RESTORE_WARNINGS } -#endif // FASTFLOAT_SSE2 +#endif // FASTFLOAT_X86_SIMD // MSVC SFINAE is broken pre-VS2017 #if defined(_MSC_VER) && _MSC_VER <= 1900 @@ -161,6 +175,244 @@ parse_four_digits_unrolled(uint32_t val) noexcept { #ifdef FASTFLOAT_HAS_SIMD +#ifdef FASTFLOAT_SSE_PATCH_REWORK + +#ifdef FASTFLOAT_X86_SIMD + +namespace detail { + +fastfloat_really_inline bool x86_all_8_digits(__m128i data) noexcept { + const __m128i zero = _mm_set1_epi8('0'); + const __m128i nine = _mm_set1_epi8('9'); + + const __m128i below = _mm_cmplt_epi8(data, zero); + const __m128i above = _mm_cmpgt_epi8(data, nine); + const __m128i bad = _mm_or_si128(below, above); +#if FASTFLOAT_X86_SIMD >= 41 + /* + * SSE4.1 PTEST. + * + * This is preferable to extracting a 16-bit mask when SSE4.1 + * is available. + */ + return _mm_testz_si128(bad, bad) != 0; +#else + return _mm_movemask_epi8(bad) == 0; +#endif +} + +fastfloat_really_inline uint32_t x86_parse_8_digits(__m128i data) noexcept { +#if FASTFLOAT_X86_SIMD >= 31 + /* + * Convert ASCII: + * + * '0'..'9' + * + * into: + * + * 0..9 + */ + data = _mm_sub_epi8(data, _mm_set1_epi8('0')); + + /* + * [d0 d1 d2 d3 d4 d5 d6 d7] + * + * PMADDUBSW: + * + * [10*d0+d1, + * 10*d2+d3, + * 10*d4+d5, + * 10*d6+d7] + */ + const __m128i pair = + _mm_maddubs_epi16(data, _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, + 10, 1, 10, 1, 10, 1)); + + /* + * PMADDWD: + * + * [12,34,56,78] + * + * -> + * + * [1234,5678] + */ + const __m128i quad = + _mm_madd_epi16(pair, _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1)); + + alignas(16) uint32_t v[4]; + _mm_store_si128(reinterpret_cast<__m128i *>(v), quad); + + return v[0] * 10000u + v[1]; +#else + /* + * SSE2 fallback. + * + * SSE3 itself has no PMADDUBSW, therefore use the existing + * SWAR algorithm. This still avoids byte-at-a-time parsing. + */ + uint64_t value; + +#if defined(FASTFLOAT_64BIT) + value = static_cast(_mm_cvtsi128_si64(data)); +#else + _mm_storel_epi64(reinterpret_cast<__m128i *>(&value), data); +#endif + + return parse_eight_digits_unrolled(value); +#endif +} + +fastfloat_really_inline bool x86_parse_if_8_digits(char const *chars, + uint64_t &value) noexcept { + FASTFLOAT_SIMD_DISABLE_WARNINGS + const __m128i data = + _mm_loadl_epi64(reinterpret_cast<__m128i const *>(chars)); + FASTFLOAT_SIMD_RESTORE_WARNINGS + if (!x86_all_8_digits(data)) + return false; + + const uint32_t digits = x86_parse_8_digits(data); + + value = value * 100000000UL + digits; + return true; +} + +fastfloat_really_inline uint32_t +x86_parse_8_digits_unchecked(char const *p) noexcept { + +#if FASTFLOAT_X86_SIMD >= 31 + + const __m128i data = _mm_loadl_epi64(reinterpret_cast(p)); + + const __m128i digits = _mm_sub_epi8(data, _mm_set1_epi8('0')); + + const __m128i pairs = + _mm_maddubs_epi16(digits, _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, + 10, 1, 10, 1, 10, 1)); + + const __m128i quads = + _mm_madd_epi16(pairs, _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1)); + + const uint32_t lo = static_cast(_mm_cvtsi128_si32(quads)); + + const uint32_t hi = static_cast(_mm_extract_epi32(quads, 1)); + + return lo * 10000U + hi; + +#else + /* + * SSE3/SSE2/32-bit: + * + * No PMADDUBSW, so use the existing SWAR + * implementation. The input is already known + * to contain digits. + */ + const uint64_t raw = read_chars_to_unsigned(p); + return parse_eight_digits_unrolled(raw); +#endif +} + +#if FASTFLOAT_X86_SIMD >= 42 + +fastfloat_really_inline bool x86_parse_if_16_digits(char const *chars, + uint64_t &value) noexcept { + FASTFLOAT_SIMD_DISABLE_WARNINGS + const __m128i data = + _mm_loadu_si128(reinterpret_cast<__m128i const *>(chars)); + FASTFLOAT_SIMD_RESTORE_WARNINGS + /* + * PCMPxSTRI range comparison. + * + * First operand: + * + * ['0','9'] + * + * Second operand: + * + * 16 input bytes + * + * Negative polarity asks for the first byte which is + * NOT inside the digit range. + */ + const __m128i ranges = + _mm_setr_epi8('0', '9', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + + const int valid = + _mm_cmpestri(ranges, 2, data, 16, + _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES | + _SIDD_NEGATIVE_POLARITY | _SIDD_LEAST_SIGNIFICANT); + + if (valid != 16) + return false; + + const uint32_t lo = x86_parse_8_digits(data); + + const uint32_t hi = x86_parse_8_digits(_mm_srli_si128(data, 8)); + + value = value * 10000000000000000UL + + static_cast(lo) * 100000000UL + hi; + + return true; +} + +fastfloat_really_inline uint64_t +x86_parse_16_digits_unchecked(char const *p) noexcept { + + const __m128i data = _mm_loadu_si128(reinterpret_cast(p)); + + const uint32_t lo = x86_parse_8_digits_unchecked(p); + + const uint32_t hi = x86_parse_8_digits_unchecked(p + 8); + + return static_cast(lo) * 100000000UL + static_cast(hi); +} + +#endif // FASTFLOAT_X86_SIMD >= 42 + +fastfloat_really_inline void +x86_parse_digits_unchecked_until_19(char const *&p, char const *end, + am_mant_t &mantissa) noexcept { + constexpr am_mant_t minimal_nineteen_digit_integer{1000000000000000000ULL}; +#if FASTFLOAT_X86_SIMD >= 42 + /* + * If mantissa has fewer than two digits, a complete + * 16-digit block cannot exceed 10^18 - 1. + */ + if (mantissa < am_mant_t(100) && (end - p) >= 16) { + const uint64_t value = x86_parse_16_digits_unchecked(p); + + mantissa = mantissa * am_mant_t(10000000000000000ULL) + + static_cast(value); + p += 16; + } +#endif + /* + * An 8-digit block is safe while: + * + * mantissa < 10^10 + * + * because the result is guaranteed < 10^18. + */ + while ((end - p) >= 8 && mantissa < am_mant_t(10000000000ULL)) { + const uint32_t value = x86_parse_8_digits_unchecked(p); + mantissa = + mantissa * am_mant_t(100000000ULL) + static_cast(value); + p += 8; + } + /* + * finalizer + */ + while (p != end && mantissa < minimal_nineteen_digit_integer) { + mantissa = mantissa * am_mant_t(10) + static_cast(*p - '0'); + ++p; + } +} +#endif +} + +#endif + // Call this if chars might not be 8 digits. // Using this style (instead of is_made_of_eight_digits_fast() then // parse_eight_digits_unrolled()) ensures we don't load SIMD registers twice. @@ -170,7 +422,7 @@ simd_parse_if_eight_digits_unrolled(char16_t const *chars, if (cpp20_and_in_constexpr()) { return false; } -#ifdef FASTFLOAT_SSE2 +#ifdef FASTFLOAT_X86_SIMD FASTFLOAT_SIMD_DISABLE_WARNINGS // Load 8 UTF-16 characters (16 bytes) // unaligned SIMD instruction -> all fine. @@ -190,7 +442,7 @@ simd_parse_if_eight_digits_unrolled(char16_t const *chars, i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data)); return true; } -#elif defined(FASTFLOAT_NEON) +#elif defined(FASTFLOAT_ARM_NEON) FASTFLOAT_SIMD_DISABLE_WARNINGS uint16x8_t const data = vld1q_u16(reinterpret_cast(chars)); FASTFLOAT_SIMD_RESTORE_WARNINGS @@ -207,7 +459,7 @@ simd_parse_if_eight_digits_unrolled(char16_t const *chars, #else (void)chars; (void)i; -#endif // FASTFLOAT_SSE2 +#endif // FASTFLOAT_X86_SIMD return false; } @@ -236,6 +488,31 @@ loop_parse_if_digits(UC const *&p, UC const *const pend, uint64_t &i) { fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void loop_parse_if_digits(char const *&p, char const *const pend, uint64_t &i) { +#if defined(FASTFLOAT_SSE_PATCH_REWORK) && defined(FASTFLOAT_X86_SIMD) +#if FASTFLOAT_X86_SIMD >= 42 + /* + * SSE4.2 handles 16 bytes at once. + * + * This is only used when a complete 16-byte block exists, + * so the load is always inside [p, pend). + */ + while (std::distance(p, pend) >= 16) { + if (!detail::x86_parse_if_16_digits(p, i)) { + break; + } + p += 16; + } +#endif + /* + * 8-byte SIMD path. + */ + while (std::distance(p, pend) >= 8 /*sizeof(uint64_t)*/) { + if (!detail::x86_parse_if_8_digits(p, i)) { + break; + } + p += 8; + } +#else // optimizes better than parse_if_eight_digits_unrolled() for UC = char. while (std::distance(p, pend) >= 8 /*sizeof(uint64_t)*/) { auto const val = read_chars_to_unsigned(p); @@ -247,6 +524,7 @@ loop_parse_if_digits(char const *&p, char const *const pend, uint64_t &i) { break; } } +#endif // Consume a remaining 4-7 digit run in a single SWAR step instead of // byte-by-byte (reuses the existing 4-digit helpers). The parsed result is // identical either way. Historically gated to clang because gcc regressed on @@ -561,11 +839,19 @@ parse_number_string(UC const *p, UC const *pend, p = answer.integer.ptr; UC const *int_end = p + answer.integer.len(); constexpr am_mant_t minimal_nineteen_digit_integer{1000000000000000000}; - do { - answer.mantissa = - answer.mantissa * 10 + static_cast(*p - UC('0')); - } while ((++p != int_end) && - (answer.mantissa < minimal_nineteen_digit_integer)); +#if defined(FASTFLOAT_SSE_PATCH_REWORK) && defined(FASTFLOAT_X86_SIMD) + if constexpr (std::is_same::value) { + detail::x86_parse_digits_unchecked_until_19(p, int_end, + answer.mantissa); + } else +#endif + { + do { + answer.mantissa = + answer.mantissa * 10 + static_cast(*p - UC('0')); + } while ((++p != int_end) && + (answer.mantissa < minimal_nineteen_digit_integer)); + } if (answer.mantissa >= minimal_nineteen_digit_integer) { // We have a big integers, so skip the fraction part completely. answer.exponent = am_pow_t(end_of_integer_part - p) + exp_number; @@ -573,11 +859,19 @@ parse_number_string(UC const *p, UC const *pend, // We have a value with a significant fractional component. p = answer.fraction.ptr; UC const *const frac_end = p + answer.fraction.len(); - while ((p != frac_end) && - (answer.mantissa < minimal_nineteen_digit_integer)) { - answer.mantissa = static_cast( - answer.mantissa * 10 + static_cast(*p - UC('0'))); - ++p; +#if defined(FASTFLOAT_SSE_PATCH_REWORK) && defined(FASTFLOAT_X86_SIMD) + if constexpr (std::is_same::value) { + detail::x86_parse_digits_unchecked_until_19(p, frac_end, + answer.mantissa); + } else +#endif + { + while ((p != frac_end) && + (answer.mantissa < minimal_nineteen_digit_integer)) { + answer.mantissa = static_cast( + answer.mantissa * 10 + static_cast(*p - UC('0'))); + ++p; + } } answer.exponent = am_pow_t(answer.fraction.ptr - p) + exp_number; } diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index 56d79c0..8bbee8b 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -192,17 +192,30 @@ using parse_options = parse_options_t; #endif #endif -#if defined(__SSE2__) || (defined(FASTFLOAT_VISUAL_STUDIO) && \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2))) -#define FASTFLOAT_SSE2 1 +#ifdef FASTFLOAT_X86_SIMD // user defined level +static_assert(FASTFLOAT_X86_SIMD >= 20 && FASTFLOAT_X86_SIMD <= 42, + "FASTFLOAT_X86_SIMD should be between 20(SSE2) and 42(SSE4.2)"); +#else // auto detect +#if defined(__SSE4_2__) +#define FASTFLOAT_X86_SIMD 42 +#elif defined(__SSE4_1__) +#define FASTFLOAT_X86_SIMD 41 +#elif defined(__SSSE3__) +#define FASTFLOAT_X86_SIMD 31 +#elif defined(__SSE3__) +#define FASTFLOAT_X86_SIMD 30 +#elif defined(__SSE2__) || (defined(FASTFLOAT_VISUAL_STUDIO) && \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2))) +#define FASTFLOAT_X86_SIMD 20 +#endif #endif #if defined(__aarch64__) || defined(_M_ARM64) -#define FASTFLOAT_NEON 1 +#define FASTFLOAT_ARM_NEON 1 #endif -#if defined(FASTFLOAT_SSE2) || defined(FASTFLOAT_NEON) +#if defined(FASTFLOAT_X86_SIMD) || defined(FASTFLOAT_ARM_NEON) #define FASTFLOAT_HAS_SIMD 1 #endif