From 604424b62468b7dfe8cea3f9ad7cdd0496b19211 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Wed, 6 Dec 2023 21:17:09 -0500 Subject: [PATCH 01/30] adding a test --- tests/basictest.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/basictest.cpp b/tests/basictest.cpp index 3d7f753..014aede 100644 --- a/tests/basictest.cpp +++ b/tests/basictest.cpp @@ -825,6 +825,7 @@ TEST_CASE("64bit.inf") { } TEST_CASE("64bit.general") { + verify("0.95000000000000000000",0.95); verify("22250738585072012e-324",0x1p-1022); /* limit between normal and subnormal*/ verify("-22250738585072012e-324",-0x1p-1022); /* limit between normal and subnormal*/ verify("-1e-999", -0.0, std::errc::result_out_of_range); From a91521f5a085ca0ed5ccde96b33b4b075961a5a2 Mon Sep 17 00:00:00 2001 From: Marvin Date: Sun, 10 Dec 2023 23:40:27 -0500 Subject: [PATCH 02/30] Added basic tests for fast_int in fast_int.cpp --- tests/fast_int.cpp | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/fast_int.cpp diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp new file mode 100644 index 0000000..78127db --- /dev/null +++ b/tests/fast_int.cpp @@ -0,0 +1,45 @@ +#include +#include +#include +#include "fast_float/fast_float.h" + +int main() +{ + + const std::vector expected_int_basic_test{0, 10, -40, 1001}; + const std::vector accept_int_basic_test {"0", "10", "-40", "1001 with text"}; + //const std::vector expected_int_basic_test{ 0, 1, -50, 9, 1001, std::numeric_limits::max(), std::numeric_limits::min()}; + //const std::vector accept_int_basic_test{ "0", "1", "-50", "9.999", "1001 with text", "2147483647", "-2147483648"}; + //const std::vector reject_int_basic_test{ "2147483648", "-2147483649", "+50"}; + + /* + const std::vector expected_int_base2_test{ 0, 1, 2, 2, 9}; + const std::vector accept_int_base2_test{ "0", "1", "10", "010", "101" }; + const std::vector reject_int_base2_test{ "2", "09" }; + + const std::vector expected_int_octal_test{ 0, 1, 7, 8}; + const std::vector accept_int_octal_test{ "00", "01", "07", "010"}; + const std::vector reject_int_octal_test{ "08", "1" }; + + const std::vector expected_int_hex_test{ 0, 1, 10, 16}; + const std::vector accept_int_hex_test{ "0", "1", "A", "0xF", "0X10"}; + const std::vector reject_int_hex_test{ "0x", "0X" }; + */ + + for (std::size_t i = 0; i < accept_int_basic_test.size(); ++i) + { + const auto& f = accept_int_basic_test[i]; + double result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to int for input:" << f << std::endl; + return EXIT_FAILURE; + } + else if (result != expected_int_basic_test[i]) { + std::cerr << "result did not match with expected int for input:" << f << std::endl; + return EXIT_FAILURE; + } + } + + return EXIT_SUCCESS; +} \ No newline at end of file From 2fb59699db16952ec256c3fbf592ed29717d5289 Mon Sep 17 00:00:00 2001 From: Marvin Date: Mon, 11 Dec 2023 01:22:48 -0500 Subject: [PATCH 03/30] Added more test cases that checks for correct errors, base 2, octal, and hex --- tests/fast_int.cpp | 131 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 108 insertions(+), 23 deletions(-) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index 78127db..8da57aa 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -5,41 +5,126 @@ int main() { +// all tests are assuming int and unsigned is size 32 bits - const std::vector expected_int_basic_test{0, 10, -40, 1001}; - const std::vector accept_int_basic_test {"0", "10", "-40", "1001 with text"}; - //const std::vector expected_int_basic_test{ 0, 1, -50, 9, 1001, std::numeric_limits::max(), std::numeric_limits::min()}; - //const std::vector accept_int_basic_test{ "0", "1", "-50", "9.999", "1001 with text", "2147483647", "-2147483648"}; - //const std::vector reject_int_basic_test{ "2147483648", "-2147483649", "+50"}; +// int basic tests + const std::vector int_basic_test_expected {0, 10, -40, 1001, 9, 2147483647, -2147483648}; + const std::vector int_basic_test {"0", "10", "-40", "1001 with text", "9.999", "2147483647 ", "-2147483648"}; - /* - const std::vector expected_int_base2_test{ 0, 1, 2, 2, 9}; - const std::vector accept_int_base2_test{ "0", "1", "10", "010", "101" }; - const std::vector reject_int_base2_test{ "2", "09" }; - - const std::vector expected_int_octal_test{ 0, 1, 7, 8}; - const std::vector accept_int_octal_test{ "00", "01", "07", "010"}; - const std::vector reject_int_octal_test{ "08", "1" }; - - const std::vector expected_int_hex_test{ 0, 1, 10, 16}; - const std::vector accept_int_hex_test{ "0", "1", "A", "0xF", "0X10"}; - const std::vector reject_int_hex_test{ "0x", "0X" }; - */ - - for (std::size_t i = 0; i < accept_int_basic_test.size(); ++i) + for (std::size_t i = 0; i < int_basic_test.size(); ++i) { - const auto& f = accept_int_basic_test[i]; - double result; + const auto& f = int_basic_test[i]; + int result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); if (answer.ec != std::errc()) { std::cerr << "could not convert to int for input:" << f << std::endl; return EXIT_FAILURE; } - else if (result != expected_int_basic_test[i]) { + else if (result != int_basic_test_expected[i]) { std::cerr << "result did not match with expected int for input:" << f << std::endl; return EXIT_FAILURE; } } +// invalid error test + const std::vector int_invalid_argument_test{ "text" , "text with 1002", "+50" " 50"}; + + for (std::size_t i = 0; i < int_invalid_argument_test.size(); ++i) + { + const auto& f = int_invalid_argument_test[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc::invalid_argument) { + std::cerr << "expected error should be 'invalid_argument'" << f << std::endl; + return EXIT_FAILURE; + } + } + + // out of range test + const std::vector int_out_of_range_test{ "2147483648", "-2147483649" }; + + for (std::size_t i = 0; i < int_out_of_range_test.size(); ++i) + { + const auto& f = int_out_of_range_test[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc::result_out_of_range) { + std::cerr << "expected error should be 'result_out_of_range'" << f << std::endl; + return EXIT_FAILURE; + } + } + + // base 2 test + const std::vector int_base_2_test_expected {0, 1, 4, 2, -1}; + const std::vector int_base_2_test {"0", "1", "100", "010", "-1"}; + + for (std::size_t i = 0; i < int_base_2_test.size(); ++i) + { + const auto& f = int_base_2_test[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 2); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to int for input:" << f << std::endl; + return EXIT_FAILURE; + } + else if (result != int_base_2_test_expected[i]) { + std::cerr << "result did not match with expected int for input:" << f << std::endl; + return EXIT_FAILURE; + } + } + + // invalid error base 2 test + const std::vector int_invalid_argument_base_2_test{ "2", "A", "-2" }; + + for (std::size_t i = 0; i < int_invalid_argument_base_2_test.size(); ++i) + { + const auto& f = int_invalid_argument_base_2_test[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 2); + if (answer.ec != std::errc::invalid_argument) { + std::cerr << "expected error should be 'invalid_argument'" << f << std::endl; + return EXIT_FAILURE; + } + } + + // octal test + const std::vector int_base_octal_test_expected {0, 1, 7, 8, 9}; + const std::vector int_base_octal_test {"0", "1", "07", "010", "0011"}; + + for (std::size_t i = 0; i < int_base_octal_test.size(); ++i) + { + const auto& f = int_base_octal_test[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 8); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to int for input:" << f << std::endl; + return EXIT_FAILURE; + } + else if (result != int_base_octal_test_expected[i]) { + std::cerr << "result did not match with expected int for input:" << f << std::endl; + return EXIT_FAILURE; + } + } + + // hex test + const std::vector int_base_hex_test_expected { 0, 1, 15, 16, 0, 16}; + const std::vector int_base_hex_test { "0", "1", "F", "010", "0x11", "10X11"}; + + for (std::size_t i = 0; i < int_base_hex_test.size(); ++i) + { + const auto& f = int_base_hex_test[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 16); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to int for input:" << f << std::endl; + return EXIT_FAILURE; + } + else if (result != int_base_hex_test_expected[i]) { + std::cerr << "result did not match with expected int for input:" << f << std::endl; + return EXIT_FAILURE; + } + } + + return EXIT_SUCCESS; } \ No newline at end of file From 122220e2f0fc9c5b5c08de135587d5b81340911c Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Mon, 11 Dec 2023 04:17:26 -0500 Subject: [PATCH 04/30] Version 1 of from_chars integer parser --- include/fast_float/ascii_number.h | 61 ++++++++++++++++++++++-- include/fast_float/fast_float.h | 7 ++- include/fast_float/float_common.h | 79 ++++++++++++++++++++++++++++++- include/fast_float/parse_number.h | 29 +++++++++--- 4 files changed, 165 insertions(+), 11 deletions(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index d18e3d5..b59e549 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "float_common.h" @@ -115,7 +116,7 @@ FASTFLOAT_SIMD_RESTORE_WARNINGS #if defined(_MSC_VER) && _MSC_VER <= 1900 template #else -template ())> +template ()) = 0> #endif // dummy for compile uint64_t simd_read8_to_u64(UC const*) { @@ -223,7 +224,7 @@ FASTFLOAT_SIMD_RESTORE_WARNINGS #if defined(_MSC_VER) && _MSC_VER <= 1900 template #else -template ())> +template ()) = 0> #endif // dummy for compile bool simd_parse_if_eight_digits_unrolled(UC const*, uint64_t&) { @@ -231,7 +232,7 @@ bool simd_parse_if_eight_digits_unrolled(UC const*, uint64_t&) { } -template ::value)> +template ::value) = 0> fastfloat_really_inline FASTFLOAT_CONSTEXPR20 void loop_parse_if_eight_digits(const UC*& p, const UC* const pend, uint64_t& i) { if (!has_simd_opt()) { @@ -439,6 +440,60 @@ parsed_number_string_t parse_number_string(UC const *p, UC const * pend, par return answer; } +template +fastfloat_really_inline FASTFLOAT_CONSTEXPR20 +from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, int base) +{ + from_chars_result_t answer; + answer.ec = std::errc::invalid_argument; + answer.ptr = p; + + bool negative = (*p == UC('-')); + if (!std::is_signed::value && negative) { + return answer; + } +#ifdef FASTFLOAT_ALLOWS_LEADING_PLUS // disabled by default + if ((*p == UC('-')) || (*p == UC('+'))) { +#else + if (*p == UC('-')) { +#endif + ++p; + } + + UC const* const start_digits = p; + + uint64_t i = 0; + while (p != pend) { + uint8_t digit = ch_to_digit(*p); + if (digit > base) { + break; + } + i = base * i + digit; // might overflow, check this later + p++; + } + + size_t digit_count = size_t(p - start_digits); + + // check u64 overflow + constexpr int max_digits = max_digits_u64(base); + if (digit_count == 0 || digit_count > max_digits) { + return answer; + } + // this check can be eliminated for all other types, but they will all require a max_digits(base) equivalent + if (digit_count == max_digits && i < min_safe_u64(base)) { + return answer; + } + + // check other types overflow + if (!std::is_same::value) { + if (i > uint64_t(std::numeric_limits::max()) + uint64_t(negative)) { + return answer; + } + } + + return negative ? (~i + 1) : i; +} + } // namespace fast_float #endif diff --git a/include/fast_float/fast_float.h b/include/fast_float/fast_float.h index 04efa87..b59fbf2 100644 --- a/include/fast_float/fast_float.h +++ b/include/fast_float/fast_float.h @@ -24,7 +24,7 @@ namespace fast_float { * to determine whether we allow the fixed point and scientific notation respectively. * The default is `fast_float::chars_format::general` which allows both `fixed` and `scientific`. */ -template +template())> FASTFLOAT_CONSTEXPR20 from_chars_result_t from_chars(UC const * first, UC const * last, T &value, chars_format fmt = chars_format::general) noexcept; @@ -36,6 +36,11 @@ template FASTFLOAT_CONSTEXPR20 from_chars_result_t from_chars_advanced(UC const * first, UC const * last, T &value, parse_options_t options) noexcept; +/** +* from_chars for integer types. +*/ +template ())> +from_chars_result_t from_chars(UC const * first, UC const * last, T& value, int base = 10) noexcept; } // namespace fast_float #include "parse_number.h" diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index d693dc3..e5d8446 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -173,7 +173,7 @@ using parse_options = parse_options_t; // rust style `try!()` macro, or `?` operator #define FASTFLOAT_TRY(x) { if (!(x)) return false; } -#define FASTFLOAT_ENABLE_IF(...) typename std::enable_if<(__VA_ARGS__), int>::type = 0 +#define FASTFLOAT_ENABLE_IF(...) typename std::enable_if<(__VA_ARGS__), int>::type namespace fast_float { @@ -186,6 +186,20 @@ fastfloat_really_inline constexpr bool cpp20_and_in_constexpr() { #endif } +template +fastfloat_really_inline constexpr bool is_supported_float_type() { + return std::is_same::value || std::is_same::value; +} + +template +fastfloat_really_inline constexpr bool is_supported_char_type() { + return + std::is_same::value || + std::is_same::value || + std::is_same::value || + std::is_same::value; +} + // Compares two ASCII strings in a case insensitive manner. template inline FASTFLOAT_CONSTEXPR14 bool @@ -674,6 +688,69 @@ constexpr char32_t const * str_const_inf() { return U"infinity"; } + + +template +struct int_luts { + static constexpr uint8_t chdigit[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 255, 255, 255, 255, 255, + 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 + }; + + static constexpr int maxdigits_u64[] = { + 64, 41, 32, 28, 25, 23, 22, 21, + 20, 19, 18, 18, 17, 17, 16, 16, + 16, 16, 15, 15, 15, 15, 14, 14, + 14, 14, 14, 14, 14, 13, 13, 13, + 13, 13, 13 + }; + + static constexpr uint64_t min_safe_u64[] = { + 9223372036854775808, 12157665459056928801, 4611686018427387904, 7450580596923828125, 4738381338321616896, + 3909821048582988049, 9223372036854775808, 12157665459056928801, 10000000000000000000, 5559917313492231481, + 2218611106740436992, 8650415919381337933, 2177953337809371136, 6568408355712890625, 1152921504606846976, + 2862423051509815793, 6746640616477458432, 15181127029874798299, 1638400000000000000, 3243919932521508681, + 6221821273427820544, 11592836324538749809, 876488338465357824, 1490116119384765625, 2481152873203736576, + 4052555153018976267, 6502111422497947648, 10260628712958602189, 15943230000000000000, 787662783788549761, + 1152921504606846976, 1667889514952984961, 2386420683693101056, 3379220508056640625, 4738381338321616896 + }; +}; + +template +constexpr uint8_t int_luts::chdigit[]; + +template +constexpr int int_luts::maxdigits_u64[]; + +template +constexpr uint64_t int_luts::min_safe_u64[]; + +template +fastfloat_really_inline +constexpr uint8_t ch_to_digit(UC c) { return int_luts<>::chdigit[c]; } + +fastfloat_really_inline +constexpr int max_digits_u64(int base) { return int_luts<>::maxdigits_u64[base - 2]; } + +// If a u64 is exactly max_digits_u64() in length, this is +// the minimum value below which it has definitely overflowed. +fastfloat_really_inline +constexpr uint64_t min_safe_u64(int base) { return int_luts<>::min_safe_u64[base - 2]; } + } // namespace fast_float #endif diff --git a/include/fast_float/parse_number.h b/include/fast_float/parse_number.h index 1c8afa4..5b74261 100644 --- a/include/fast_float/parse_number.h +++ b/include/fast_float/parse_number.h @@ -133,7 +133,7 @@ fastfloat_really_inline bool rounds_to_nearest() noexcept { } // namespace detail -template +template FASTFLOAT_CONSTEXPR20 from_chars_result_t from_chars(UC const * first, UC const * last, T &value, chars_format fmt /*= chars_format::general*/) noexcept { @@ -145,11 +145,8 @@ FASTFLOAT_CONSTEXPR20 from_chars_result_t from_chars_advanced(UC const * first, UC const * last, T &value, parse_options_t options) noexcept { - static_assert (std::is_same::value || std::is_same::value, "only float and double are supported"); - static_assert (std::is_same::value || - std::is_same::value || - std::is_same::value || - std::is_same::value , "only char, wchar_t, char16_t and char32_t are supported"); + static_assert (is_supported_float_type(), "only float and double are supported"); + static_assert (is_supported_char_type(), "only char, wchar_t, char16_t and char32_t are supported"); from_chars_result_t answer; #ifdef FASTFLOAT_SKIP_WHITE_SPACE // disabled by default @@ -232,6 +229,26 @@ from_chars_result_t from_chars_advanced(UC const * first, UC const * last, return answer; } + +template +from_chars_result_t from_chars(UC const* first, UC const* last, T& value, int base) noexcept +{ + static_assert (is_supported_char_type(), "only char, wchar_t, char16_t and char32_t are supported"); + + from_chars_result_t answer; +#ifdef FASTFLOAT_SKIP_WHITE_SPACE // disabled by default + while ((first != last) && fast_float::is_space(uint8_t(*first))) { + first++; + } +#endif + if (first == last) { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + return answer; + } + return parse_int_string(first, last, base); +} + } // namespace fast_float #endif From c9527c2e4f78759fa516bc122f229c07ff23c1e0 Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Mon, 11 Dec 2023 04:27:22 -0500 Subject: [PATCH 05/30] Skip leading zeros --- include/fast_float/ascii_number.h | 5 +++++ include/fast_float/float_common.h | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index b59e549..333ada9 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -460,6 +460,11 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, ++p; } + // skip leading zeros + while (p != pend && *p == UC('0')) { + ++p; + } + UC const* const start_digits = p; uint64_t i = 0; diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index e5d8446..f0d09cb 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -747,7 +747,7 @@ fastfloat_really_inline constexpr int max_digits_u64(int base) { return int_luts<>::maxdigits_u64[base - 2]; } // If a u64 is exactly max_digits_u64() in length, this is -// the minimum value below which it has definitely overflowed. +// the value below which it has definitely overflowed. fastfloat_really_inline constexpr uint64_t min_safe_u64(int base) { return int_luts<>::min_safe_u64[base - 2]; } From e60b47455b354092bc9afe883fca39c34c37c678 Mon Sep 17 00:00:00 2001 From: Marvin Date: Mon, 11 Dec 2023 18:17:33 -0500 Subject: [PATCH 06/30] Added test cases that coverunsigned, out or range errors, char pointers, invalid bases, and out of range bases --- tests/fast_int.cpp | 315 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 282 insertions(+), 33 deletions(-) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index 8da57aa..8b88f2c 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -1,15 +1,34 @@ #include #include #include +#include +#include #include "fast_float/fast_float.h" +/* +all tests conducted are to check fast_float::from_chars functionality with int and unsigned +test cases include: +int basic tests - numbers only, numbers with strings behind, decimals, negative numbers +unsigned basic tests - numbers only, numbers with strings behind, decimals +int invalid tests - strings only, strings with numbers behind, space in front of number, plus sign in front of number +unsigned invalid tests - strings only, strings with numbers behind, space in front of number, plus/minus sign in front of number +int out of range tests - numbers exceeding int bit size (Note: out of range errors for 8, 16, 32, and 64 bits have not been tested) +unsigned out of range tests - numbers exceeding unsigned bit size (Note: out of range errors for 8, 16, 32, and 64 bits have not been tested) +int pointer tests - points to first character that is not recognized as int +unsigned pointer tests - points to first character that is not recognized as unsigned +int/unsigned base 2 tests - numbers are converted from binary to decimal +octal tests - numbers are converted from octal to decimal +hex tests - numbers are converted from hex to decimal (Note: 0x and 0X are considered invalid) +invalid base tests - everything is invalid +out of range base tests - should still work even with a base greater than 36 +*/ + int main() { -// all tests are assuming int and unsigned is size 32 bits -// int basic tests - const std::vector int_basic_test_expected {0, 10, -40, 1001, 9, 2147483647, -2147483648}; - const std::vector int_basic_test {"0", "10", "-40", "1001 with text", "9.999", "2147483647 ", "-2147483648"}; + // int basic test + const std::vector int_basic_test_expected { 0, 10, -40, 1001, 9 }; + const std::vector int_basic_test { "0", "10 ", "-40", "1001 with text", "9.999" }; for (std::size_t i = 0; i < int_basic_test.size(); ++i) { @@ -17,17 +36,36 @@ int main() int result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); if (answer.ec != std::errc()) { - std::cerr << "could not convert to int for input:" << f << std::endl; + std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; return EXIT_FAILURE; } else if (result != int_basic_test_expected[i]) { - std::cerr << "result did not match with expected int for input:" << f << std::endl; + std::cerr << "result " << std::quoted(f) << " did not match with expected int: " << int_basic_test_expected[i] << std::endl; return EXIT_FAILURE; } } -// invalid error test - const std::vector int_invalid_argument_test{ "text" , "text with 1002", "+50" " 50"}; + // unsigned basic test + const std::vector unsigned_basic_test_expected { 0, 10, 1001, 9 }; + const std::vector unsigned_basic_test { "0", "10 ", "1001 with text", "9.999" }; + + for (std::size_t i = 0; i < unsigned_basic_test.size(); ++i) + { + const auto& f = unsigned_basic_test[i]; + unsigned result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to unsigned for input: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + else if (result != unsigned_basic_test_expected[i]) { + std::cerr << "result " << std::quoted(f) << " did not match with expected unsigned: " << unsigned_basic_test_expected[i] << std::endl; + return EXIT_FAILURE; + } + } + + // int invalid error test + const std::vector int_invalid_argument_test{ "text", "text with 1002", "+50", " 50" }; for (std::size_t i = 0; i < int_invalid_argument_test.size(); ++i) { @@ -35,13 +73,27 @@ int main() int result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); if (answer.ec != std::errc::invalid_argument) { - std::cerr << "expected error should be 'invalid_argument'" << f << std::endl; + std::cerr << "expected error should be 'invalid_argument' for: " << std::quoted(f) << std::endl; return EXIT_FAILURE; } } - // out of range test - const std::vector int_out_of_range_test{ "2147483648", "-2147483649" }; + // unsigned invalid error test + const std::vector unsigned_invalid_argument_test{ "text", "text with 1002", "+50", " 50", "-50" }; + + for (std::size_t i = 0; i < unsigned_invalid_argument_test.size(); ++i) + { + const auto& f = unsigned_invalid_argument_test[i]; + unsigned result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc::invalid_argument) { + std::cerr << "expected error should be 'invalid_argument' for: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + } + + // int out of range error test + const std::vector int_out_of_range_test{ "2000000000000000000000" }; for (std::size_t i = 0; i < int_out_of_range_test.size(); ++i) { @@ -49,14 +101,146 @@ int main() int result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); if (answer.ec != std::errc::result_out_of_range) { - std::cerr << "expected error should be 'result_out_of_range'" << f << std::endl; + std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; return EXIT_FAILURE; } } - // base 2 test - const std::vector int_base_2_test_expected {0, 1, 4, 2, -1}; - const std::vector int_base_2_test {"0", "1", "100", "010", "-1"}; + // unsigned out of range error test + const std::vector unsigned_out_of_range_test{ "2000000000000000000000" }; + + for (std::size_t i = 0; i < unsigned_out_of_range_test.size(); ++i) + { + const auto& f = unsigned_out_of_range_test[i]; + unsigned result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc::result_out_of_range) { + std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + } + + // int pointer test #1 (only numbers) + const std::vector int_pointer_test_1 { "0", "010", "-40" }; + + for (std::size_t i = 0; i < int_pointer_test_1.size(); ++i) + { + const auto& f = int_pointer_test_1[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + else if (strcmp(answer.ptr, "") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted("") << std::endl; + return EXIT_FAILURE; + } + } + + // int pointer test #2 (string behind numbers) + const std::vector int_pointer_test_2 { "1001 with text" }; + + for (std::size_t i = 0; i < int_pointer_test_2.size(); ++i) + { + const auto& f = int_pointer_test_2[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + else if (strcmp(answer.ptr, " with text") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; + return EXIT_FAILURE; + } + } + + // int pointer test #3 (string with newline behind numbers) + const std::vector int_pointer_test_3 { "1001 with text\n" }; + + for (std::size_t i = 0; i < int_pointer_test_3.size(); ++i) + { + const auto& f = int_pointer_test_3[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + else if (strcmp(answer.ptr, " with text\n") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; + return EXIT_FAILURE; + } + } + + // int pointer test #4 (float) + const std::vector int_pointer_test_4 { "9.999" }; + + for (std::size_t i = 0; i < int_pointer_test_4.size(); ++i) + { + const auto& f = int_pointer_test_4[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + else if (strcmp(answer.ptr, ".999") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(".999") << std::endl; + return EXIT_FAILURE; + } + } + + // int pointer test #5 (invalid int) + const std::vector int_pointer_test_5 { "+50" }; + + for (std::size_t i = 0; i < int_pointer_test_5.size(); ++i) + { + const auto& f = int_pointer_test_5[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (strcmp(answer.ptr, "+50") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted("+50") << std::endl; + return EXIT_FAILURE; + } + } + + // unsigned pointer test #1 (string behind numbers) + const std::vector unsigned_pointer_test_1 { "1001 with text" }; + + for (std::size_t i = 0; i < unsigned_pointer_test_1.size(); ++i) + { + const auto& f = unsigned_pointer_test_1[i]; + unsigned result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to unsigned for input: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + if (strcmp(answer.ptr, " with text") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; + return EXIT_FAILURE; + } + } + + // unsigned pointer test #2 (invalid unsigned) + const std::vector unsigned_pointer_test_2 { "-50" }; + + for (std::size_t i = 0; i < unsigned_pointer_test_2.size(); ++i) + { + const auto& f = unsigned_pointer_test_2[i]; + unsigned result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (strcmp(answer.ptr, "-50") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted("-50") << std::endl; + return EXIT_FAILURE; + } + } + + // int base 2 test + const std::vector int_base_2_test_expected { 0, 1, 4, 2, -1 }; + const std::vector int_base_2_test { "0", "1", "100", "010", "-1" }; for (std::size_t i = 0; i < int_base_2_test.size(); ++i) { @@ -64,16 +248,35 @@ int main() int result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 2); if (answer.ec != std::errc()) { - std::cerr << "could not convert to int for input:" << f << std::endl; + std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; return EXIT_FAILURE; } else if (result != int_base_2_test_expected[i]) { - std::cerr << "result did not match with expected int for input:" << f << std::endl; + std::cerr << "result " << std::quoted(f) << " did not match with expected int: " << int_base_2_test_expected[i] << std::endl; return EXIT_FAILURE; } } - // invalid error base 2 test + // unsigned base 2 test + const std::vector unsigned_base_2_test_expected { 0, 1, 4, 2 }; + const std::vector unsigned_base_2_test { "0", "1", "100", "010" }; + + for (std::size_t i = 0; i < unsigned_base_2_test.size(); ++i) + { + const auto& f = unsigned_base_2_test[i]; + unsigned result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 2); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to unsigned for input: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + else if (result != unsigned_base_2_test_expected[i]) { + std::cerr << "result " << std::quoted(f) << " did not match with expected unsigned: " << unsigned_base_2_test_expected[i] << std::endl; + return EXIT_FAILURE; + } + } + + // int invalid error base 2 test const std::vector int_invalid_argument_base_2_test{ "2", "A", "-2" }; for (std::size_t i = 0; i < int_invalid_argument_base_2_test.size(); ++i) @@ -82,49 +285,95 @@ int main() int result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 2); if (answer.ec != std::errc::invalid_argument) { - std::cerr << "expected error should be 'invalid_argument'" << f << std::endl; + std::cerr << "expected error should be 'invalid_argument' for: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + } + + // unsigned invalid error base 2 test + const std::vector unsigned_invalid_argument_base_2_test{ "2", "A", "-1", "-2" }; + + for (std::size_t i = 0; i < unsigned_invalid_argument_base_2_test.size(); ++i) + { + const auto& f = unsigned_invalid_argument_base_2_test[i]; + unsigned result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 2); + if (answer.ec != std::errc::invalid_argument) { + std::cerr << "expected error should be 'invalid_argument' for: " << std::quoted(f) << std::endl; return EXIT_FAILURE; } } // octal test - const std::vector int_base_octal_test_expected {0, 1, 7, 8, 9}; - const std::vector int_base_octal_test {"0", "1", "07", "010", "0011"}; + const std::vector base_octal_test_expected {0, 1, 7, 8, 9}; + const std::vector base_octal_test { "0", "1", "07", "010", "0011" }; - for (std::size_t i = 0; i < int_base_octal_test.size(); ++i) + for (std::size_t i = 0; i < base_octal_test.size(); ++i) { - const auto& f = int_base_octal_test[i]; + const auto& f = base_octal_test[i]; int result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 8); if (answer.ec != std::errc()) { - std::cerr << "could not convert to int for input:" << f << std::endl; + std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; return EXIT_FAILURE; } - else if (result != int_base_octal_test_expected[i]) { - std::cerr << "result did not match with expected int for input:" << f << std::endl; + else if (result != base_octal_test_expected[i]) { + std::cerr << "result " << std::quoted(f) << " did not match with expected int: " << base_octal_test_expected[i] << std::endl; return EXIT_FAILURE; } } // hex test - const std::vector int_base_hex_test_expected { 0, 1, 15, 16, 0, 16}; - const std::vector int_base_hex_test { "0", "1", "F", "010", "0x11", "10X11"}; + const std::vector base_hex_test_expected { 0, 1, 15, 16, 0, 16}; + const std::vector base_hex_test { "0", "1", "F", "01f", "0x11", "10X11" }; - for (std::size_t i = 0; i < int_base_hex_test.size(); ++i) + for (std::size_t i = 0; i < base_hex_test.size(); ++i) { - const auto& f = int_base_hex_test[i]; + const auto& f = base_hex_test[i]; int result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 16); if (answer.ec != std::errc()) { - std::cerr << "could not convert to int for input:" << f << std::endl; + std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; return EXIT_FAILURE; } - else if (result != int_base_hex_test_expected[i]) { - std::cerr << "result did not match with expected int for input:" << f << std::endl; + else if (result != base_hex_test_expected[i]) { + std::cerr << "result " << std::quoted(f) << " did not match with expected int: " << base_hex_test_expected[i] << std::endl; return EXIT_FAILURE; } } + // invalid base test (-1) + const std::vector invalid_base_test { "0", "1", "-1", "F", "10Z" }; + + for (std::size_t i = 0; i < invalid_base_test.size(); ++i) + { + const auto& f = invalid_base_test[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, -1); + if (answer.ec != std::errc::invalid_argument) { + std::cerr << "expected error should be 'invalid_argument' for: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + } + + // out of range base test (100) + const std::vector base_out_of_range_test_expected { 0, 1, 15, 35, 10035 }; + const std::vector base_out_of_range_test { "0", "1", "F", "Z", "10Z" }; + + for (std::size_t i = 0; i < base_out_of_range_test.size(); ++i) + { + const auto& f = base_out_of_range_test[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 100); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + else if (result != base_out_of_range_test_expected[i]) { + std::cerr << "result " << std::quoted(f) << " did not match with expected int: " << base_out_of_range_test_expected[i] << std::endl; + return EXIT_FAILURE; + } + } return EXIT_SUCCESS; } \ No newline at end of file From 3d446f1ebafcfe567f5350cfabe4a6728980268b Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Mon, 11 Dec 2023 04:40:08 -0500 Subject: [PATCH 07/30] Fix gcc werrors --- include/fast_float/ascii_number.h | 25 +++++++++++++++++++------ include/fast_float/fast_float.h | 1 + include/fast_float/float_common.h | 16 ++++++++-------- include/fast_float/parse_number.h | 3 ++- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index 333ada9..322fba7 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -445,11 +445,11 @@ fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, int base) { from_chars_result_t answer; - answer.ec = std::errc::invalid_argument; - answer.ptr = p; bool negative = (*p == UC('-')); if (!std::is_signed::value && negative) { + answer.ec = std::errc::invalid_argument; + answer.ptr = p; return answer; } #ifdef FASTFLOAT_ALLOWS_LEADING_PLUS // disabled by default @@ -473,14 +473,16 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, if (digit > base) { break; } - i = base * i + digit; // might overflow, check this later + i = uint64_t(base) * i + digit; // might overflow, check this later p++; } size_t digit_count = size_t(p - start_digits); + answer.ec = std::errc::result_out_of_range; + answer.ptr = p; // check u64 overflow - constexpr int max_digits = max_digits_u64(base); + size_t max_digits = max_digits_u64(base); if (digit_count == 0 || digit_count > max_digits) { return answer; } @@ -495,8 +497,19 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, return answer; } } - - return negative ? (~i + 1) : i; + + if (negative) { + // this weird workaround is required because: + // - converting unsigned to signed when its value is greater than signed max is UB pre-C++23. + // - reinterpret_cast(~i + 1) would have worked, but it is not constexpr + // this should be optimized away. + value = -std::numeric_limits::max() - + static_cast(i - std::numeric_limits::max()); + } + else value = T(i); + + answer.ec = std::errc(); + return answer; } } // namespace fast_float diff --git a/include/fast_float/fast_float.h b/include/fast_float/fast_float.h index b59fbf2..9b1b9b4 100644 --- a/include/fast_float/fast_float.h +++ b/include/fast_float/fast_float.h @@ -40,6 +40,7 @@ from_chars_result_t from_chars_advanced(UC const * first, UC const * last, * from_chars for integer types. */ template ())> +FASTFLOAT_CONSTEXPR20 from_chars_result_t from_chars(UC const * first, UC const * last, T& value, int base = 10) noexcept; } // namespace fast_float diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index f0d09cb..0721d12 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -711,7 +711,7 @@ struct int_luts { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 }; - static constexpr int maxdigits_u64[] = { + static constexpr size_t maxdigits_u64[] = { 64, 41, 32, 28, 25, 23, 22, 21, 20, 19, 18, 18, 17, 17, 16, 16, 16, 16, 15, 15, 15, 15, 14, 14, @@ -720,12 +720,12 @@ struct int_luts { }; static constexpr uint64_t min_safe_u64[] = { - 9223372036854775808, 12157665459056928801, 4611686018427387904, 7450580596923828125, 4738381338321616896, - 3909821048582988049, 9223372036854775808, 12157665459056928801, 10000000000000000000, 5559917313492231481, + 9223372036854775808ull, 12157665459056928801ull, 4611686018427387904, 7450580596923828125, 4738381338321616896, + 3909821048582988049, 9223372036854775808ull, 12157665459056928801ull, 10000000000000000000ull, 5559917313492231481, 2218611106740436992, 8650415919381337933, 2177953337809371136, 6568408355712890625, 1152921504606846976, - 2862423051509815793, 6746640616477458432, 15181127029874798299, 1638400000000000000, 3243919932521508681, - 6221821273427820544, 11592836324538749809, 876488338465357824, 1490116119384765625, 2481152873203736576, - 4052555153018976267, 6502111422497947648, 10260628712958602189, 15943230000000000000, 787662783788549761, + 2862423051509815793, 6746640616477458432, 15181127029874798299ull, 1638400000000000000, 3243919932521508681, + 6221821273427820544, 11592836324538749809ull, 876488338465357824, 1490116119384765625, 2481152873203736576, + 4052555153018976267, 6502111422497947648, 10260628712958602189ull, 15943230000000000000ull, 787662783788549761, 1152921504606846976, 1667889514952984961, 2386420683693101056, 3379220508056640625, 4738381338321616896 }; }; @@ -734,7 +734,7 @@ template constexpr uint8_t int_luts::chdigit[]; template -constexpr int int_luts::maxdigits_u64[]; +constexpr size_t int_luts::maxdigits_u64[]; template constexpr uint64_t int_luts::min_safe_u64[]; @@ -744,7 +744,7 @@ fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) { return int_luts<>::chdigit[c]; } fastfloat_really_inline -constexpr int max_digits_u64(int base) { return int_luts<>::maxdigits_u64[base - 2]; } +constexpr size_t max_digits_u64(int base) { return int_luts<>::maxdigits_u64[base - 2]; } // If a u64 is exactly max_digits_u64() in length, this is // the value below which it has definitely overflowed. diff --git a/include/fast_float/parse_number.h b/include/fast_float/parse_number.h index 5b74261..4c75f5d 100644 --- a/include/fast_float/parse_number.h +++ b/include/fast_float/parse_number.h @@ -231,6 +231,7 @@ from_chars_result_t from_chars_advanced(UC const * first, UC const * last, template +FASTFLOAT_CONSTEXPR20 from_chars_result_t from_chars(UC const* first, UC const* last, T& value, int base) noexcept { static_assert (is_supported_char_type(), "only char, wchar_t, char16_t and char32_t are supported"); @@ -246,7 +247,7 @@ from_chars_result_t from_chars(UC const* first, UC const* last, T& value, in answer.ptr = first; return answer; } - return parse_int_string(first, last, base); + return parse_int_string(first, last, value, base); } } // namespace fast_float From 7a21a8d6d734c54529ec108c64a518b87fde5736 Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Tue, 12 Dec 2023 02:36:18 -0500 Subject: [PATCH 08/30] Return invalid_argument in more places --- include/fast_float/ascii_number.h | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index 322fba7..3dc8b43 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -445,11 +445,13 @@ fastfloat_really_inline FASTFLOAT_CONSTEXPR20 from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, int base) { from_chars_result_t answer; + + UC const* const first = p; bool negative = (*p == UC('-')); if (!std::is_signed::value && negative) { answer.ec = std::errc::invalid_argument; - answer.ptr = p; + answer.ptr = first; return answer; } #ifdef FASTFLOAT_ALLOWS_LEADING_PLUS // disabled by default @@ -478,22 +480,31 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, } size_t digit_count = size_t(p - start_digits); - answer.ec = std::errc::result_out_of_range; + + if (digit_count == 0) { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + return answer; + } + answer.ptr = p; // check u64 overflow size_t max_digits = max_digits_u64(base); - if (digit_count == 0 || digit_count > max_digits) { - return answer; + if (digit_count > max_digits) { + answer.ec = std::errc::result_out_of_range; + return answer; } // this check can be eliminated for all other types, but they will all require a max_digits(base) equivalent - if (digit_count == max_digits && i < min_safe_u64(base)) { + if (digit_count == max_digits && i < min_safe_u64(base)) { + answer.ec = std::errc::result_out_of_range; return answer; } // check other types overflow if (!std::is_same::value) { if (i > uint64_t(std::numeric_limits::max()) + uint64_t(negative)) { + answer.ec = std::errc::result_out_of_range; return answer; } } From 20c9375c5eb71841148782cb9ae5609d36b26ab1 Mon Sep 17 00:00:00 2001 From: Marvin Date: Tue, 12 Dec 2023 13:59:18 -0500 Subject: [PATCH 09/30] Added new test cases for out of range errors that cover 8,16,32,64 bits, out of range errors for all bases (64 bit only), and fixed some test cases --- tests/CMakeLists.txt | 2 +- tests/fast_int.cpp | 398 +++++++++++++++++++++++++++++++------------ 2 files changed, 293 insertions(+), 107 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0739a00..2cf649c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -71,7 +71,7 @@ endif() fast_float_add_cpp_test(long_test) fast_float_add_cpp_test(powersoffive_hardround) fast_float_add_cpp_test(string_test) - +fast_float_add_cpp_test(fast_int) fast_float_add_cpp_test(json_fmt) fast_float_add_cpp_test(fortran) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index 8b88f2c..2d8576d 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -4,6 +4,7 @@ #include #include #include "fast_float/fast_float.h" +#include /* all tests conducted are to check fast_float::from_chars functionality with int and unsigned @@ -12,15 +13,15 @@ int basic tests - numbers only, numbers with strings behind, decimals, negative unsigned basic tests - numbers only, numbers with strings behind, decimals int invalid tests - strings only, strings with numbers behind, space in front of number, plus sign in front of number unsigned invalid tests - strings only, strings with numbers behind, space in front of number, plus/minus sign in front of number -int out of range tests - numbers exceeding int bit size (Note: out of range errors for 8, 16, 32, and 64 bits have not been tested) -unsigned out of range tests - numbers exceeding unsigned bit size (Note: out of range errors for 8, 16, 32, and 64 bits have not been tested) +int out of range tests - numbers exceeding int bit size for 8, 16, 32, and 64 bits +unsigned out of range tests - numbers exceeding unsigned bit size 8, 16, 32, and 64 bits int pointer tests - points to first character that is not recognized as int unsigned pointer tests - points to first character that is not recognized as unsigned int/unsigned base 2 tests - numbers are converted from binary to decimal octal tests - numbers are converted from octal to decimal hex tests - numbers are converted from hex to decimal (Note: 0x and 0X are considered invalid) -invalid base tests - everything is invalid -out of range base tests - should still work even with a base greater than 36 +invalid base tests - any base not within 2-36 is invalid +out of range base tests - numbers exceeding int/unsigned bit size after converted from base (Note: only 64 bit int and unsigned are tested) */ int main() @@ -92,13 +93,13 @@ int main() } } - // int out of range error test - const std::vector int_out_of_range_test{ "2000000000000000000000" }; + // int out of range error test #1 (8 bit) + const std::vector int_out_of_range_test_1{ "2000000000000000000000", "128", "-129"}; - for (std::size_t i = 0; i < int_out_of_range_test.size(); ++i) + for (std::size_t i = 0; i < int_out_of_range_test_1.size(); ++i) { - const auto& f = int_out_of_range_test[i]; - int result; + const auto& f = int_out_of_range_test_1[i]; + int8_t result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); if (answer.ec != std::errc::result_out_of_range) { std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; @@ -106,13 +107,97 @@ int main() } } - // unsigned out of range error test - const std::vector unsigned_out_of_range_test{ "2000000000000000000000" }; + // int out of range error test #2 (16 bit) + const std::vector int_out_of_range_test_2{ "2000000000000000000000", "32768", "-32769"}; - for (std::size_t i = 0; i < unsigned_out_of_range_test.size(); ++i) + for (std::size_t i = 0; i < int_out_of_range_test_2.size(); ++i) { - const auto& f = unsigned_out_of_range_test[i]; - unsigned result; + const auto& f = int_out_of_range_test_2[i]; + int16_t result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc::result_out_of_range) { + std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + } + + // int out of range error test #3 (32 bit) + const std::vector int_out_of_range_test_3{ "2000000000000000000000", "2147483648", "-2147483649"}; + + for (std::size_t i = 0; i < int_out_of_range_test_3.size(); ++i) + { + const auto& f = int_out_of_range_test_3[i]; + int32_t result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc::result_out_of_range) { + std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + } + + // int out of range error test #4 (64 bit) + const std::vector int_out_of_range_test_4{ "2000000000000000000000", "9223372036854775808", "-9223372036854775809"}; + + for (std::size_t i = 0; i < int_out_of_range_test_4.size(); ++i) + { + const auto& f = int_out_of_range_test_4[i]; + int64_t result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc::result_out_of_range) { + std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + } + + // unsigned out of range error test #1 (8 bit) + const std::vector unsigned_out_of_range_test_1{ "2000000000000000000000", "255" }; + + for (std::size_t i = 0; i < unsigned_out_of_range_test_1.size(); ++i) + { + const auto& f = unsigned_out_of_range_test_1[i]; + uint8_t result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc::result_out_of_range) { + std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + } + + // unsigned out of range error test #2 (16 bit) + const std::vector unsigned_out_of_range_test_2{ "2000000000000000000000", "65536" }; + + for (std::size_t i = 0; i < unsigned_out_of_range_test_2.size(); ++i) + { + const auto& f = unsigned_out_of_range_test_2[i]; + uint16_t result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc::result_out_of_range) { + std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + } + + // unsigned out of range error test #3 (32 bit) + const std::vector unsigned_out_of_range_test_3{ "2000000000000000000000", "4294967296" }; + + for (std::size_t i = 0; i < unsigned_out_of_range_test_3.size(); ++i) + { + const auto& f = unsigned_out_of_range_test_3[i]; + uint32_t result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc::result_out_of_range) { + std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + } + + // unsigned out of range error test #4 (64 bit) + const std::vector unsigned_out_of_range_test_4{ "2000000000000000000000", "18446744073709551616" }; + + for (std::size_t i = 0; i < unsigned_out_of_range_test_4.size(); ++i) + { + const auto& f = unsigned_out_of_range_test_4[i]; + uint64_t result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); if (answer.ec != std::errc::result_out_of_range) { std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; @@ -139,103 +224,69 @@ int main() } // int pointer test #2 (string behind numbers) - const std::vector int_pointer_test_2 { "1001 with text" }; + const std::string int_pointer_test_2 = "1001 with text"; - for (std::size_t i = 0; i < int_pointer_test_2.size(); ++i) - { - const auto& f = int_pointer_test_2[i]; - int result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (answer.ec != std::errc()) { - std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; - return EXIT_FAILURE; - } - else if (strcmp(answer.ptr, " with text") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; - return EXIT_FAILURE; - } + const auto& f = int_pointer_test_2; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (strcmp(answer.ptr, " with text") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; + return EXIT_FAILURE; } // int pointer test #3 (string with newline behind numbers) - const std::vector int_pointer_test_3 { "1001 with text\n" }; + const std::string int_pointer_test_3 = "1001 with text\n"; - for (std::size_t i = 0; i < int_pointer_test_3.size(); ++i) - { - const auto& f = int_pointer_test_3[i]; - int result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (answer.ec != std::errc()) { - std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; - return EXIT_FAILURE; - } - else if (strcmp(answer.ptr, " with text\n") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; - return EXIT_FAILURE; - } + const auto& f = int_pointer_test_3; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (strcmp(answer.ptr, " with text\n") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; + return EXIT_FAILURE; } // int pointer test #4 (float) - const std::vector int_pointer_test_4 { "9.999" }; + const std::string int_pointer_test_4 = "9.999"; - for (std::size_t i = 0; i < int_pointer_test_4.size(); ++i) - { - const auto& f = int_pointer_test_4[i]; - int result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (answer.ec != std::errc()) { - std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; - return EXIT_FAILURE; - } - else if (strcmp(answer.ptr, ".999") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(".999") << std::endl; - return EXIT_FAILURE; - } + const auto& f = int_pointer_test_4; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (strcmp(answer.ptr, ".999") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(".999") << std::endl; + return EXIT_FAILURE; } // int pointer test #5 (invalid int) - const std::vector int_pointer_test_5 { "+50" }; + const std::string int_pointer_test_5 = "+50"; - for (std::size_t i = 0; i < int_pointer_test_5.size(); ++i) - { - const auto& f = int_pointer_test_5[i]; - int result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (strcmp(answer.ptr, "+50") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted("+50") << std::endl; - return EXIT_FAILURE; - } + const auto& f = int_pointer_test_5; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (strcmp(answer.ptr, "+50") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted("+50") << std::endl; + return EXIT_FAILURE; } // unsigned pointer test #1 (string behind numbers) - const std::vector unsigned_pointer_test_1 { "1001 with text" }; + const std::string unsigned_pointer_test_1 = "1001 with text"; - for (std::size_t i = 0; i < unsigned_pointer_test_1.size(); ++i) - { - const auto& f = unsigned_pointer_test_1[i]; - unsigned result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (answer.ec != std::errc()) { - std::cerr << "could not convert to unsigned for input: " << std::quoted(f) << std::endl; - return EXIT_FAILURE; - } - if (strcmp(answer.ptr, " with text") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; - return EXIT_FAILURE; - } + const auto& f = unsigned_pointer_test_1; + unsigned result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (strcmp(answer.ptr, " with text") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; + return EXIT_FAILURE; } // unsigned pointer test #2 (invalid unsigned) - const std::vector unsigned_pointer_test_2 { "-50" }; + const std::string unsigned_pointer_test_2 = "-50"; - for (std::size_t i = 0; i < unsigned_pointer_test_2.size(); ++i) - { - const auto& f = unsigned_pointer_test_2[i]; - unsigned result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (strcmp(answer.ptr, "-50") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted("-50") << std::endl; - return EXIT_FAILURE; - } + const auto& f = unsigned_pointer_test_2; + unsigned result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (strcmp(answer.ptr, "-50") != 0) { + std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted("-50") << std::endl; + return EXIT_FAILURE; } // int base 2 test @@ -342,12 +393,12 @@ int main() } } - // invalid base test (-1) - const std::vector invalid_base_test { "0", "1", "-1", "F", "10Z" }; + // invalid base test #1 (-1) + const std::vector invalid_base_test_1 { "0", "1", "-1", "F", "10Z" }; - for (std::size_t i = 0; i < invalid_base_test.size(); ++i) + for (std::size_t i = 0; i < invalid_base_test_1.size(); ++i) { - const auto& f = invalid_base_test[i]; + const auto& f = invalid_base_test_1[i]; int result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, -1); if (answer.ec != std::errc::invalid_argument) { @@ -356,24 +407,159 @@ int main() } } - // out of range base test (100) - const std::vector base_out_of_range_test_expected { 0, 1, 15, 35, 10035 }; - const std::vector base_out_of_range_test { "0", "1", "F", "Z", "10Z" }; + // invalid base test #2 (37) + const std::vector invalid_base_test_2 { "0", "1", "F", "Z", "10Z" }; - for (std::size_t i = 0; i < base_out_of_range_test.size(); ++i) + for (std::size_t i = 0; i < invalid_base_test_2.size(); ++i) { - const auto& f = base_out_of_range_test[i]; + const auto& f = invalid_base_test_2[i]; int result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 100); - if (answer.ec != std::errc()) { - std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; - return EXIT_FAILURE; - } - else if (result != base_out_of_range_test_expected[i]) { - std::cerr << "result " << std::quoted(f) << " did not match with expected int: " << base_out_of_range_test_expected[i] << std::endl; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 37); + if (answer.ec != std::errc::invalid_argument) { + std::cerr << "expected error should be 'invalid_argument' for: " << std::quoted(f) << std::endl; return EXIT_FAILURE; } } + // int out of range error base test (64 bit) + const std::vector int_out_of_range_base_test { "1000000000000000000000000000000000000000000000000000000000000000", + "-1000000000000000000000000000000000000000000000000000000000000001", + "2021110011022210012102010021220101220222", + "-2021110011022210012102010021220101221000", + "20000000000000000000000000000000", + "-20000000000000000000000000000001", + "1104332401304422434310311213", + "-1104332401304422434310311214", + "1540241003031030222122212", + "-1540241003031030222122213" + "22341010611245052052301", + "-22341010611245052052302" + "1000000000000000000000", + "-1000000000000000000001", + "67404283172107811828", + "-67404283172107811830", + "9223372036854775808", + "-9223372036854775809", + "1728002635214590698", + "-1728002635214590699", + "41A792678515120368", + "-41A792678515120369", + "10B269549075433C38", + "-10B269549075433C39", + "4340724C6C71DC7A8", + "-4340724C6C71DC7A9", + "160E2AD3246366808", + "-160E2AD3246366809", + "8000000000000000", + "-8000000000000001", + "33D3D8307B214009", + "-33D3D8307B21400A", + "16AGH595DF825FA8", + "-16AGH595DF825FA9", + "BA643DCI0FFEEHI", + "-BA643DCI0FFEEI0" + "5CBFJIA3FH26JA8", + "-5CBFJIA3FH26JA9", + "2HEICIIIE82DH98", + "-2HEICIIIE82DH99", + "1ADAIBB21DCKFA8", + "-1ADAIBB21DCKFA9", + "I6K448CF4192C3", + "-I6K448CF4192C4", + "ACD772JNC9L0L8", + "-ACD772JNC9L0L9", + "64IE1FOCNN5G78", + "-64IE1FOCNN5G79", + "3IGOECJBMCA688", + "-3IGOECJBMCA689", + "27C48L5B37OAOQ", + "-27C48L5B37OAP0", + "1BK39F3AH3DMQ8", + "-1BK39F3AH3DMQ9", + "Q1SE8F0M04ISC", + "-Q1SE8F0M04ISD", + "HAJPPBC1FC208", + "-HAJPPBC1FC209", + "BM03I95HIA438", + "-BM03I95HIA439", + "8000000000000", + "-8000000000001" + "5HG4CK9JD4U38", + "-5HG4CK9JD4U39", + "3TDTK1V8J6TPQ", + "-3TDTK1V8J6TPR", + "2PIJMIKEXRXP8", + "-2PIJMIKEXRXP9", + "1Y2P0IJ32E8E8", + "-1Y2P0IJ32E8E9" }; + int base = 2; + int counter = 0; + for (std::size_t i = 0; i < int_out_of_range_base_test.size(); ++i) + { + const auto& f = int_out_of_range_base_test[i]; + int64_t result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, base); + if (answer.ec != std::errc::result_out_of_range) { + std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + if (!(counter)) { + ++counter; + } + else { + ++base; + ++counter; + } + } + + // unsigned out of range error base test (64 bit) + const std::vector unsigned_out_of_range_base_test { "10000000000000000000000000000000000000000000000000000000000000000", + "11112220022122120101211020120210210211221", + "100000000000000000000000000000000", + "2214220303114400424121122431", + "3520522010102100444244424", + "45012021522523134134602", + "2000000000000000000000", + "145808576354216723757", + "18446744073709551616", + "335500516A429071285", + "839365134A2A240714", + "219505A9511A867B73", + "8681049ADB03DB172", + "2C1D56B648C6CD111", + "10000000000000000", + "67979G60F5428011", + "2D3FGB0B9CG4BD2G", + "141C8786H1CCAAGH", + "B53BJH07BE4DJ0G", + "5E8G4GGG7G56DIG", + "2L4LF104353J8KG", + "1DDH88H2782I516", + "L12EE5FN0JI1IG", + "C9C336O0MLB7EG", + "7B7N2PCNIOKCGG", + "4EO8HFAM6FLLMP", + "2NC6J26L66RHOG", + "1N3RSH11F098RO", + "14L9LKMO30O40G", + "ND075IB45K86G", + "G000000000000", + "B1W8P7J5Q9R6G", + "7ORP63SH4DPHI", + "5G24A25TWKWFG", + "3W5E11264SGSG" }; + int base = 2; + for (std::size_t i = 0; i < unsigned_out_of_range_base_test.size(); ++i) + { + const auto& f = unsigned_out_of_range_base_test[i]; + uint64_t result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, base); + if (answer.ec != std::errc::result_out_of_range) { + std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + ++base; + } + return EXIT_SUCCESS; } \ No newline at end of file From e4702e039f4b95b4be06b80b8507a80090c1601d Mon Sep 17 00:00:00 2001 From: TheRandomGuy146275 Date: Tue, 12 Dec 2023 17:58:32 -0500 Subject: [PATCH 10/30] Fixing cmake errrors --- tests/fast_int.cpp | 74 +++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index 2d8576d..219e457 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -226,66 +226,66 @@ int main() // int pointer test #2 (string behind numbers) const std::string int_pointer_test_2 = "1001 with text"; - const auto& f = int_pointer_test_2; - int result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (strcmp(answer.ptr, " with text") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; + const auto& f2 = int_pointer_test_2; + int result2; + auto answer2 = fast_float::from_chars(f2.data(), f2.data() + f2.size(), result2); + if (strcmp(answer2.ptr, " with text") != 0) { + std::cerr << "ptr of result " << std::quoted(f2) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; return EXIT_FAILURE; } // int pointer test #3 (string with newline behind numbers) const std::string int_pointer_test_3 = "1001 with text\n"; - const auto& f = int_pointer_test_3; - int result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (strcmp(answer.ptr, " with text\n") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; + const auto& f3 = int_pointer_test_3; + int result3; + auto answer3 = fast_float::from_chars(f3.data(), f3.data() + f3.size(), result3); + if (strcmp(answer3.ptr, " with text\n") != 0) { + std::cerr << "ptr of result " << std::quoted(f3) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; return EXIT_FAILURE; } // int pointer test #4 (float) const std::string int_pointer_test_4 = "9.999"; - const auto& f = int_pointer_test_4; - int result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (strcmp(answer.ptr, ".999") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(".999") << std::endl; + const auto& f4 = int_pointer_test_4; + int result4; + auto answer4 = fast_float::from_chars(f4.data(), f4.data() + f4.size(), result4); + if (strcmp(answer4.ptr, ".999") != 0) { + std::cerr << "ptr of result " << std::quoted(f4) << " did not match with expected ptr: " << std::quoted(".999") << std::endl; return EXIT_FAILURE; } // int pointer test #5 (invalid int) const std::string int_pointer_test_5 = "+50"; - const auto& f = int_pointer_test_5; - int result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (strcmp(answer.ptr, "+50") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted("+50") << std::endl; + const auto& f5 = int_pointer_test_5; + int result5; + auto answer5 = fast_float::from_chars(f5.data(), f5.data() + f5.size(), result5); + if (strcmp(answer5.ptr, "+50") != 0) { + std::cerr << "ptr of result " << std::quoted(f5) << " did not match with expected ptr: " << std::quoted("+50") << std::endl; return EXIT_FAILURE; } - // unsigned pointer test #1 (string behind numbers) + // unsigned pointer test #2 (string behind numbers) const std::string unsigned_pointer_test_1 = "1001 with text"; - const auto& f = unsigned_pointer_test_1; - unsigned result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (strcmp(answer.ptr, " with text") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; + const auto& f6 = unsigned_pointer_test_1; + unsigned result6; + auto answer6 = fast_float::from_chars(f6.data(), f6.data() + f6.size(), result6); + if (strcmp(answer6.ptr, " with text") != 0) { + std::cerr << "ptr of result " << std::quoted(f6) << " did not match with expected ptr: " << std::quoted(" with text") << std::endl; return EXIT_FAILURE; } // unsigned pointer test #2 (invalid unsigned) const std::string unsigned_pointer_test_2 = "-50"; - const auto& f = unsigned_pointer_test_2; - unsigned result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); - if (strcmp(answer.ptr, "-50") != 0) { - std::cerr << "ptr of result " << std::quoted(f) << " did not match with expected ptr: " << std::quoted("-50") << std::endl; + const auto& f7 = unsigned_pointer_test_2; + unsigned result7; + auto answer7 = fast_float::from_chars(f7.data(), f7.data() + f7.size(), result7); + if (strcmp(answer7.ptr, "-50") != 0) { + std::cerr << "ptr of result " << std::quoted(f7) << " did not match with expected ptr: " << std::quoted("-50") << std::endl; return EXIT_FAILURE; } @@ -492,13 +492,13 @@ int main() "-2PIJMIKEXRXP9", "1Y2P0IJ32E8E8", "-1Y2P0IJ32E8E9" }; - int base = 2; + int base_int = 2; int counter = 0; for (std::size_t i = 0; i < int_out_of_range_base_test.size(); ++i) { const auto& f = int_out_of_range_base_test[i]; int64_t result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, base); + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, base_int); if (answer.ec != std::errc::result_out_of_range) { std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; return EXIT_FAILURE; @@ -507,7 +507,7 @@ int main() ++counter; } else { - ++base; + ++base_int; ++counter; } } @@ -548,17 +548,17 @@ int main() "7ORP63SH4DPHI", "5G24A25TWKWFG", "3W5E11264SGSG" }; - int base = 2; + int base_unsigned = 2; for (std::size_t i = 0; i < unsigned_out_of_range_base_test.size(); ++i) { const auto& f = unsigned_out_of_range_base_test[i]; uint64_t result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, base); + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, base_unsigned); if (answer.ec != std::errc::result_out_of_range) { std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; return EXIT_FAILURE; } - ++base; + ++base_unsigned; } return EXIT_SUCCESS; From 5fda2cc240ab38ab5e6145edadb6a4a2389b1288 Mon Sep 17 00:00:00 2001 From: Marvin Date: Tue, 12 Dec 2023 19:07:51 -0500 Subject: [PATCH 11/30] Debugging results --- tests/fast_int.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index 219e457..2304918 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -36,8 +36,9 @@ int main() const auto& f = int_basic_test[i]; int result; auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); + if (answer.ec != std::errc()) { - std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; + std::cerr << "1. could not convert to int for input: " << std::quoted(f) << " " << result << " " << answer.ptr << std::endl; return EXIT_FAILURE; } else if (result != int_basic_test_expected[i]) { From 681eb1ea38fff131dac092db5aa870b79f3bd8d1 Mon Sep 17 00:00:00 2001 From: Marvin Date: Tue, 12 Dec 2023 19:18:07 -0500 Subject: [PATCH 12/30] More details to basic test error for debugging --- tests/fast_int.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index 2304918..3f1ef72 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -38,7 +38,15 @@ int main() auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result); if (answer.ec != std::errc()) { - std::cerr << "1. could not convert to int for input: " << std::quoted(f) << " " << result << " " << answer.ptr << std::endl; + if (answer.ec == std::errc::invalid_argument) { + std::cerr << "could not convert to int for input: " << std::quoted(f) << "because of invalid arguement, output: " << result << " , ptr: " << answer.ptr << std::endl; + } + else if (answer.ec == std::errc::result_out_of_range) { + std::cerr << "could not convert to int for input: " << std::quoted(f) << "because it's out of range, output: " << result << " , ptr: " << answer.ptr << std::endl; + } + else { + std::cerr << "could not convert to int for input: " << std::quoted(f) << "because of an unknown error, output: " << result << " , ptr: " << answer.ptr << std::endl; + } return EXIT_FAILURE; } else if (result != int_basic_test_expected[i]) { From 0711006266485105f53bf4357c09c2a7d0d1742f Mon Sep 17 00:00:00 2001 From: Marvin Date: Tue, 12 Dec 2023 19:23:43 -0500 Subject: [PATCH 13/30] Fixed messages --- tests/fast_int.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index 3f1ef72..5811b58 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -39,13 +39,13 @@ int main() if (answer.ec != std::errc()) { if (answer.ec == std::errc::invalid_argument) { - std::cerr << "could not convert to int for input: " << std::quoted(f) << "because of invalid arguement, output: " << result << " , ptr: " << answer.ptr << std::endl; + std::cerr << "could not convert to int for input: " << std::quoted(f) << " because of invalid arguement, output: " << result << " , ptr: " << answer.ptr << std::endl; } else if (answer.ec == std::errc::result_out_of_range) { - std::cerr << "could not convert to int for input: " << std::quoted(f) << "because it's out of range, output: " << result << " , ptr: " << answer.ptr << std::endl; + std::cerr << "could not convert to int for input: " << std::quoted(f) << " because it's out of range, output: " << result << " , ptr: " << answer.ptr << std::endl; } else { - std::cerr << "could not convert to int for input: " << std::quoted(f) << "because of an unknown error, output: " << result << " , ptr: " << answer.ptr << std::endl; + std::cerr << "could not convert to int for input: " << std::quoted(f) << " because of an unknown error, output: " << result << " , ptr: " << answer.ptr << std::endl; } return EXIT_FAILURE; } From ebc15bec518a37ef664817130eb7147ef9b038d0 Mon Sep 17 00:00:00 2001 From: Marvin Date: Tue, 12 Dec 2023 20:20:05 -0500 Subject: [PATCH 14/30] Added test case for numbers within range after converted from base --- tests/fast_int.cpp | 141 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index 5811b58..6509143 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -22,6 +22,7 @@ octal tests - numbers are converted from octal to decimal hex tests - numbers are converted from hex to decimal (Note: 0x and 0X are considered invalid) invalid base tests - any base not within 2-36 is invalid out of range base tests - numbers exceeding int/unsigned bit size after converted from base (Note: only 64 bit int and unsigned are tested) +within range base tests - max/min numbers are still within int/unsigned bit size after converted from base (Note: only 64 bit int and unsigned are tested) */ int main() @@ -570,5 +571,145 @@ int main() ++base_unsigned; } + // just within range base test (64 bit) + const std::vector int_within_range_base_test { "111111111111111111111111111111111111111111111111111111111111111", + "-1000000000000000000000000000000000000000000000000000000000000000", + "2021110011022210012102010021220101220221", + "-2021110011022210012102010021220101220222", + "13333333333333333333333333333333", + "-20000000000000000000000000000000", + "1104332401304422434310311212", + "-1104332401304422434310311213", + "1540241003031030222122211", + "-1540241003031030222122212" + "22341010611245052052300", + "-22341010611245052052301" + "777777777777777777777", + "-1000000000000000000000", + "67404283172107811827", + "-67404283172107811828", + "9223372036854775807", + "-9223372036854775808", + "1728002635214590697", + "-1728002635214590698", + "41A792678515120367", + "-41A792678515120368", + "10B269549075433C37", + "-10B269549075433C38", + "4340724C6C71DC7A7", + "-4340724C6C71DC7A8", + "160E2AD3246366807", + "-160E2AD3246366808", + "7FFFFFFFFFFFFFFF", + "-8000000000000000", + "33D3D8307B214008", + "-33D3D8307B214009", + "16AGH595DF825FA7", + "-16AGH595DF825FA8", + "BA643DCI0FFEEHH", + "-BA643DCI0FFEEHI" + "5CBFJIA3FH26JA7", + "-5CBFJIA3FH26JA8", + "2HEICIIIE82DH97", + "-2HEICIIIE82DH98", + "1ADAIBB21DCKFA7", + "-1ADAIBB21DCKFA8", + "I6K448CF4192C2", + "-I6K448CF4192C3", + "ACD772JNC9L0L7", + "-ACD772JNC9L0L8", + "64IE1FOCNN5G77", + "-64IE1FOCNN5G78", + "3IGOECJBMCA687", + "-3IGOECJBMCA688", + "27C48L5B37OAOP", + "-27C48L5B37OAOQ", + "1BK39F3AH3DMQ7", + "-1BK39F3AH3DMQ8", + "Q1SE8F0M04ISB", + "-Q1SE8F0M04ISC", + "HAJPPBC1FC207", + "-HAJPPBC1FC208", + "BM03I95HIA437", + "-BM03I95HIA438", + "7VVVVVVVVVVVV", + "-8000000000000" + "5HG4CK9JD4U37", + "-5HG4CK9JD4U38", + "3TDTK1V8J6TPP", + "-3TDTK1V8J6TPQ", + "2PIJMIKEXRXP7", + "-2PIJMIKEXRXP8", + "1Y2P0IJ32E8E7", + "-1Y2P0IJ32E8E8" }; + int base_int2 = 2; + int counter2 = 0; + for (std::size_t i = 0; i < int_within_range_base_test.size(); ++i) + { + const auto& f = int_within_range_base_test[i]; + int64_t result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, base_int2); + if (answer.ec != std::errc()) { + std::cerr << "converting " << std::quoted(f) << " to int failed (most likely out of range)" << std::endl; + return EXIT_FAILURE; + } + if (!(counter2)) { + ++counter2; + } + else { + ++base_int2; + ++counter2; + } + } + + // unsigned within range base test (64 bit) + const std::vector unsigned_within_range_base_test { "1111111111111111111111111111111111111111111111111111111111111111", + "11112220022122120101211020120210210211220", + "33333333333333333333333333333333", + "2214220303114400424121122430", + "3520522010102100444244423", + "45012021522523134134601", + "1777777777777777777777", + "145808576354216723756", + "18446744073709551615", + "335500516A429071284", + "839365134A2A240713", + "219505A9511A867B72", + "8681049ADB03DB171", + "2C1D56B648C6CD110", + "FFFFFFFFFFFFFFFF", + "67979G60F5428010", + "2D3FGB0B9CG4BD2F", + "141C8786H1CCAAGG", + "B53BJH07BE4DJ0F", + "5E8G4GGG7G56DIF", + "2L4LF104353J8KF", + "1DDH88H2782I515", + "L12EE5FN0JI1IF", + "C9C336O0MLB7EF", + "7B7N2PCNIOKCGF", + "4EO8HFAM6FLLMO", + "2NC6J26L66RHOF", + "1N3RSH11F098RO", + "14L9LKMO30O40F", + "ND075IB45K86F", + "FVVVVVVVVVVVV", + "B1W8P7J5Q9R6F", + "7ORP63SH4DPHH", + "5G24A25TWKWFF", + "3W5E11264SGSF" }; + int base_unsigned2 = 2; + for (std::size_t i = 0; i < unsigned_within_range_base_test.size(); ++i) + { + const auto& f = unsigned_within_range_base_test[i]; + uint64_t result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, base_unsigned2); + if (answer.ec != std::errc()) { + std::cerr << "converting " << std::quoted(f) << " to unsigned failed (most likely out of range)" << std::endl; + return EXIT_FAILURE; + } + ++base_unsigned2; + } + return EXIT_SUCCESS; } \ No newline at end of file From 624ba49434d24b81117ef41d6c2b5ff3a1d05668 Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Tue, 12 Dec 2023 21:26:48 -0500 Subject: [PATCH 15/30] Fix more Werrors - Werror=conversion,char-subscripts --- include/fast_float/ascii_number.h | 13 ++++--------- include/fast_float/float_common.h | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index 3dc8b43..5c82c6c 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -7,6 +7,7 @@ #include #include #include +#include #include "float_common.h" @@ -462,11 +463,6 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, ++p; } - // skip leading zeros - while (p != pend && *p == UC('0')) { - ++p; - } - UC const* const start_digits = p; uint64_t i = 0; @@ -512,10 +508,9 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, if (negative) { // this weird workaround is required because: // - converting unsigned to signed when its value is greater than signed max is UB pre-C++23. - // - reinterpret_cast(~i + 1) would have worked, but it is not constexpr - // this should be optimized away. - value = -std::numeric_limits::max() - - static_cast(i - std::numeric_limits::max()); + // - reinterpret_casting (~i + 1) would work, but it is not constexpr + // this is always optimized into a neg instruction. + value = T(-std::numeric_limits::max() - T(i - std::numeric_limits::max())); } else value = T(i); diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index 0721d12..3595491 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -741,7 +741,7 @@ constexpr uint64_t int_luts::min_safe_u64[]; template fastfloat_really_inline -constexpr uint8_t ch_to_digit(UC c) { return int_luts<>::chdigit[c]; } +constexpr uint8_t ch_to_digit(UC c) { return int_luts<>::chdigit[static_cast(c)]; } fastfloat_really_inline constexpr size_t max_digits_u64(int base) { return int_luts<>::maxdigits_u64[base - 2]; } From 01e8c50a3317f05155636d9270174eae9118f8c8 Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Tue, 12 Dec 2023 22:35:14 -0500 Subject: [PATCH 16/30] Fix bugs in tests/fast_int.cpp --- tests/fast_int.cpp | 44 ++++++++++++++------------------------------ 1 file changed, 14 insertions(+), 30 deletions(-) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index b107d59..1e197ed 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -385,7 +385,7 @@ int main() } // hex test - const std::vector base_hex_test_expected { 0, 1, 15, 16, 0, 16}; + const std::vector base_hex_test_expected { 0, 1, 15, 31, 0, 16}; const std::vector base_hex_test { "0", "1", "F", "01f", "0x11", "10X11" }; for (std::size_t i = 0; i < base_hex_test.size(); ++i) @@ -441,9 +441,9 @@ int main() "1104332401304422434310311213", "-1104332401304422434310311214", "1540241003031030222122212", - "-1540241003031030222122213" + "-1540241003031030222122213", "22341010611245052052301", - "-22341010611245052052302" + "-22341010611245052052302", "1000000000000000000000", "-1000000000000000000001", "67404283172107811828", @@ -467,7 +467,7 @@ int main() "16AGH595DF825FA8", "-16AGH595DF825FA9", "BA643DCI0FFEEHI", - "-BA643DCI0FFEEI0" + "-BA643DCI0FFEEI0", "5CBFJIA3FH26JA8", "-5CBFJIA3FH26JA9", "2HEICIIIE82DH98", @@ -493,7 +493,7 @@ int main() "BM03I95HIA438", "-BM03I95HIA439", "8000000000000", - "-8000000000001" + "-8000000000001", "5HG4CK9JD4U38", "-5HG4CK9JD4U39", "3TDTK1V8J6TPQ", @@ -502,24 +502,16 @@ int main() "-2PIJMIKEXRXP9", "1Y2P0IJ32E8E8", "-1Y2P0IJ32E8E9" }; - int base_int = 2; - int counter = 0; + for (std::size_t i = 0; i < int_out_of_range_base_test.size(); ++i) { const auto& f = int_out_of_range_base_test[i]; int64_t result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, base_int); + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 2 + (i / 2)); if (answer.ec != std::errc::result_out_of_range) { std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; return EXIT_FAILURE; } - if (!(counter)) { - ++counter; - } - else { - ++base_int; - ++counter; - } } // unsigned out of range error base test (64 bit) @@ -581,9 +573,9 @@ int main() "1104332401304422434310311212", "-1104332401304422434310311213", "1540241003031030222122211", - "-1540241003031030222122212" + "-1540241003031030222122212", "22341010611245052052300", - "-22341010611245052052301" + "-22341010611245052052301", "777777777777777777777", "-1000000000000000000000", "67404283172107811827", @@ -607,7 +599,7 @@ int main() "16AGH595DF825FA7", "-16AGH595DF825FA8", "BA643DCI0FFEEHH", - "-BA643DCI0FFEEHI" + "-BA643DCI0FFEEHI", "5CBFJIA3FH26JA7", "-5CBFJIA3FH26JA8", "2HEICIIIE82DH97", @@ -633,7 +625,7 @@ int main() "BM03I95HIA437", "-BM03I95HIA438", "7VVVVVVVVVVVV", - "-8000000000000" + "-8000000000000", "5HG4CK9JD4U37", "-5HG4CK9JD4U38", "3TDTK1V8J6TPP", @@ -642,24 +634,16 @@ int main() "-2PIJMIKEXRXP8", "1Y2P0IJ32E8E7", "-1Y2P0IJ32E8E8" }; - int base_int2 = 2; - int counter2 = 0; + for (std::size_t i = 0; i < int_within_range_base_test.size(); ++i) { const auto& f = int_within_range_base_test[i]; int64_t result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, base_int2); + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 2 + (i / 2)); if (answer.ec != std::errc()) { std::cerr << "converting " << std::quoted(f) << " to int failed (most likely out of range)" << std::endl; return EXIT_FAILURE; } - if (!(counter2)) { - ++counter2; - } - else { - ++base_int2; - ++counter2; - } } // unsigned within range base test (64 bit) @@ -690,7 +674,7 @@ int main() "7B7N2PCNIOKCGF", "4EO8HFAM6FLLMO", "2NC6J26L66RHOF", - "1N3RSH11F098RO", + "1N3RSH11F098RN", "14L9LKMO30O40F", "ND075IB45K86F", "FVVVVVVVVVVVV", From a30fe866f6d94c0d2c84478173e173d891ecd544 Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Tue, 12 Dec 2023 22:35:58 -0500 Subject: [PATCH 17/30] Fix bugs highlighted in tests --- include/fast_float/ascii_number.h | 2 +- include/fast_float/parse_number.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index 5c82c6c..f37241c 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -468,7 +468,7 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, uint64_t i = 0; while (p != pend) { uint8_t digit = ch_to_digit(*p); - if (digit > base) { + if (digit >= base) { break; } i = uint64_t(base) * i + digit; // might overflow, check this later diff --git a/include/fast_float/parse_number.h b/include/fast_float/parse_number.h index 4c75f5d..1fd587b 100644 --- a/include/fast_float/parse_number.h +++ b/include/fast_float/parse_number.h @@ -242,7 +242,7 @@ from_chars_result_t from_chars(UC const* first, UC const* last, T& value, in first++; } #endif - if (first == last) { + if (first == last || base < 2 || base > 36) { answer.ec = std::errc::invalid_argument; answer.ptr = first; return answer; From 30b3165520ff7b509a2a243e570f49b30814dc4d Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Tue, 12 Dec 2023 22:45:14 -0500 Subject: [PATCH 18/30] Fix fast_int test Werrors --- tests/fast_int.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index 1e197ed..af61bce 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -40,13 +40,13 @@ int main() if (answer.ec != std::errc()) { if (answer.ec == std::errc::invalid_argument) { - std::cerr << "could not convert to int for input: " << std::quoted(f) << " because of invalid arguement, output: " << result << " , ptr: " << answer.ptr << std::endl; + std::cerr << "could not convert to int for input: " << std::quoted(f) << " because of invalid arguement" << std::endl; } else if (answer.ec == std::errc::result_out_of_range) { - std::cerr << "could not convert to int for input: " << std::quoted(f) << " because it's out of range, output: " << result << " , ptr: " << answer.ptr << std::endl; + std::cerr << "could not convert to int for input: " << std::quoted(f) << " because it's out of range" << std::endl; } else { - std::cerr << "could not convert to int for input: " << std::quoted(f) << " because of an unknown error, output: " << result << " , ptr: " << answer.ptr << std::endl; + std::cerr << "could not convert to int for input: " << std::quoted(f) << " because of an unknown error" << std::endl; } return EXIT_FAILURE; } @@ -507,7 +507,7 @@ int main() { const auto& f = int_out_of_range_base_test[i]; int64_t result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 2 + (i / 2)); + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, int(2 + (i / 2))); if (answer.ec != std::errc::result_out_of_range) { std::cerr << "expected error for should be 'result_out_of_range': " << std::quoted(f) << std::endl; return EXIT_FAILURE; @@ -639,7 +639,7 @@ int main() { const auto& f = int_within_range_base_test[i]; int64_t result; - auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, 2 + (i / 2)); + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, int(2 + (i / 2))); if (answer.ec != std::errc()) { std::cerr << "converting " << std::quoted(f) << " to int failed (most likely out of range)" << std::endl; return EXIT_FAILURE; From 1e0a9da5382f552b2134975d951056c4ade0ddc6 Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Wed, 13 Dec 2023 00:56:35 -0500 Subject: [PATCH 19/30] Change FASTLOAT_ARM64 macro to FASTFLOAT_NEON --- include/fast_float/float_common.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index d693dc3..85a5a1f 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -134,7 +134,7 @@ using parse_options = parse_options_t; #define FASTFLOAT_NEON 1 #endif -#if defined(FASTFLOAT_SSE2) || defined(FASTFLOAT_ARM64) +#if defined(FASTFLOAT_SSE2) || defined(FASTFLOAT_NEON) #define FASTFLOAT_HAS_SIMD 1 #endif From 36aaded3dd8acd0e901a411c726b814c160da090 Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Wed, 13 Dec 2023 17:11:41 -0500 Subject: [PATCH 20/30] Fix handling of leading zeros --- include/fast_float/ascii_number.h | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index f37241c..0fc12d2 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -463,6 +463,12 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, ++p; } + UC const* const start_num = p; + while (*p == UC('0')) { + ++p; + } + const bool has_leading_zeros = p > start_num; + UC const* const start_digits = p; uint64_t i = 0; @@ -478,8 +484,15 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, size_t digit_count = size_t(p - start_digits); if (digit_count == 0) { - answer.ec = std::errc::invalid_argument; - answer.ptr = first; + if (has_leading_zeros) { + value = 0; + answer.ec = std::errc(); + answer.ptr = p; + } + else { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + } return answer; } From 26a5b2eb16518fb1aae6d49b6aedf547328f8dec Mon Sep 17 00:00:00 2001 From: TheRandomGuy146275 <135289032+TheRandomGuy146275@users.noreply.github.com> Date: Wed, 13 Dec 2023 17:13:32 -0500 Subject: [PATCH 21/30] Added test case for ignoring leading zeros for all bases - added: fix incorrect base for leading zeros test --------- Co-authored-by: Marvin Co-authored-by: Maya Warrier <34803055+mayawarrier@users.noreply.github.com> --- tests/fast_int.cpp | 53 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/fast_int.cpp b/tests/fast_int.cpp index af61bce..85dbca8 100644 --- a/tests/fast_int.cpp +++ b/tests/fast_int.cpp @@ -23,6 +23,7 @@ hex tests - numbers are converted from hex to decimal (Note: 0x and 0X are consi invalid base tests - any base not within 2-36 is invalid out of range base tests - numbers exceeding int/unsigned bit size after converted from base (Note: only 64 bit int and unsigned are tested) within range base tests - max/min numbers are still within int/unsigned bit size after converted from base (Note: only 64 bit int and unsigned are tested) +leading zeros tests - ignores all zeroes in front of valid number after converted from base */ int main() @@ -695,5 +696,57 @@ int main() ++base_unsigned2; } + // int leading zeros test + const std::vector int_leading_zeros_test { "00000000000000000000000000000000000000000000000000000000000000000000001111110111", + "000000000000000000000000000000000000000000000000001101121", + "000000000000000000000000000000000000000033313", + "00000000000000000000000000000013030", + "0000000000000000000000000000004411", + "0000000000000000000000000000002650", + "0000000000000000000000000000001767", + "0000000000000000000000000000001347", + "0000000000000000000000000000001015", + "00000000000000000000843", + "00000000000000000000707", + "00000000000000000000601", + "00000000000000000000527", + "0000000000000000000047A", + "000000000000000000003F7", + "0000000000000000000038C", + "00000000000000000000327", + "000000000000000000002F8", + "000000000000000000002AF", + "00000000000000000000267", + "00000000000000000000223", + "000000000000000000001L3", + "000000000000000000001I7", + "000000000000000000001FF", + "000000000000000000001D1", + "000000000000000000001AG", + "00000000000000000000187", + "00000000000000000000160", + "0000000000000000000013P", + "0000000000000000000011N", + "00000000000000000000VN", + "00000000000000000000UP", + "00000000000000000000TT", + "00000000000000000000T0", + "00000000000000000000S7" }; + + for (std::size_t i = 0; i < int_leading_zeros_test.size(); ++i) + { + const auto& f = int_leading_zeros_test[i]; + int result; + auto answer = fast_float::from_chars(f.data(), f.data() + f.size(), result, int(i + 2)); + if (answer.ec != std::errc()) { + std::cerr << "could not convert to int for input: " << std::quoted(f) << std::endl; + return EXIT_FAILURE; + } + else if (result != 1015) { + std::cerr << "result " << std::quoted(f) << " did not match with expected int: " << 1015 << std::endl; + return EXIT_FAILURE; + } + } + return EXIT_SUCCESS; } \ No newline at end of file From bdee16bcad3ad035aa53a156a5e5b8a92f57e278 Mon Sep 17 00:00:00 2001 From: Maya Warrier Date: Wed, 13 Dec 2023 17:42:30 -0500 Subject: [PATCH 22/30] - Add SIMD acceleration to fast_int - fix MSVC warning --- include/fast_float/ascii_number.h | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index 0fc12d2..3641f02 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -7,7 +7,6 @@ #include #include #include -#include #include "float_common.h" @@ -472,6 +471,9 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, UC const* const start_digits = p; uint64_t i = 0; + if (base == 10) { + loop_parse_if_eight_digits(p, pend, i); // use SIMD if possible + } while (p != pend) { uint8_t digit = ch_to_digit(*p); if (digit >= base) { @@ -519,11 +521,18 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, } if (negative) { +#ifdef FASTFLOAT_VISUAL_STUDIO +#pragma warning(push) +#pragma warning(disable: 4146) +#endif // this weird workaround is required because: // - converting unsigned to signed when its value is greater than signed max is UB pre-C++23. // - reinterpret_casting (~i + 1) would work, but it is not constexpr // this is always optimized into a neg instruction. value = T(-std::numeric_limits::max() - T(i - std::numeric_limits::max())); +#ifdef FASTFLOAT_VISUAL_STUDIO +#pragma warning(pop) +#endif } else value = T(i); From 882a716c127073ed081d2a47ae5765654f768f01 Mon Sep 17 00:00:00 2001 From: Maya Warrier <34803055+mayawarrier@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:28:23 -0500 Subject: [PATCH 23/30] Explicit curly bracket where suggested Co-authored-by: Daniel Lemire --- include/fast_float/ascii_number.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index 3641f02..5d3eac9 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -534,7 +534,7 @@ from_chars_result_t parse_int_string(UC const* p, UC const* pend, T& value, #pragma warning(pop) #endif } - else value = T(i); + else { value = T(i); } answer.ec = std::errc(); return answer; From 8b849ebab0de4da1f64cc29dea9967adfcace6cc Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 14 Dec 2023 17:53:30 -0500 Subject: [PATCH 24/30] Documentation regarding the integer types --- README.md | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1ca0b7e..2028051 100644 --- a/README.md +++ b/README.md @@ -4,17 +4,21 @@ [![Ubuntu 22.04 CI (GCC 11)](https://github.com/fastfloat/fast_float/actions/workflows/ubuntu22.yml/badge.svg)](https://github.com/fastfloat/fast_float/actions/workflows/ubuntu22.yml) The fast_float library provides fast header-only implementations for the C++ from_chars -functions for `float` and `double` types. These functions convert ASCII strings representing -decimal values (e.g., `1.3e10`) into binary types. We provide exact rounding (including +functions for `float` and `double` types as well as integer types. These functions convert ASCII strings representing decimal values (e.g., `1.3e10`) into binary types. We provide exact rounding (including round to even). In our experience, these `fast_float` functions many times faster than comparable number-parsing functions from existing C++ standard libraries. -Specifically, `fast_float` provides the following two functions with a C++17-like syntax (the library itself only requires C++11): +Specifically, `fast_float` provides the following two functions to parse floating-point numbers with a C++17-like syntax (the library itself only requires C++11): ```C++ from_chars_result from_chars(const char* first, const char* last, float& value, ...); from_chars_result from_chars(const char* first, const char* last, double& value, ...); ``` +You can also parse integer types: + + + + The return type (`from_chars_result`) is defined as the struct: ```C++ struct from_chars_result { @@ -103,6 +107,43 @@ We support Visual Studio, macOS, Linux, freeBSD. We support big and little endia We assume that the rounding mode is set to nearest (`std::fegetround() == FE_TONEAREST`). + +## Integer types + +You can also parse integer types using different bases (e.g., 2, 10, 16). The following code will +print the number 22250738585072012 three times: + + +```C++ + uint64_t i; + const char str[] = "22250738585072012"; + auto answer = fast_float::from_chars(str, str + strlen(str), i); + if (answer.ec != std::errc()) { + std::cerr << "parsing failure\n"; + return EXIT_FAILURE; + } + std::cout << "parsed the number "<< i << std::endl; + + const char binstr[] = "1001111000011001110110111001001010110100111000110001100"; + + answer = fast_float::from_chars(binstr, binstr + strlen(binstr), i, 2); + if (answer.ec != std::errc()) { + std::cerr << "parsing failure\n"; + return EXIT_FAILURE; + } + std::cout << "parsed the number "<< i << std::endl; + + + const char hexstr[] = "4f0cedc95a718c"; + + answer = fast_float::from_chars(hexstr, hexstr + strlen(hexstr), i, 16); + if (answer.ec != std::errc()) { + std::cerr << "parsing failure\n"; + return EXIT_FAILURE; + } + std::cout << "parsed the number "<< i << std::endl; +``` + ## C++20: compile-time evaluation (constexpr) In C++20, you may use `fast_float::from_chars` to parse strings From e59e8f547a4a29a7fc04d2b5a52e2f254dec13fb Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 14 Dec 2023 17:54:39 -0500 Subject: [PATCH 25/30] adding space --- CONTRIBUTORS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index e339869..ef91743 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -6,4 +6,4 @@ Tim Paine Fabio Pellacini Lénárd Szolnoki Jan Pharago -Maya Warrier \ No newline at end of file +Maya Warrier From f3ff46fd40d28b5eb272727812e5bd8dc5a6d876 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 14 Dec 2023 17:56:17 -0500 Subject: [PATCH 26/30] version bump --- CMakeLists.txt | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8847f06..4431a8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.9) -project(fast_float VERSION 5.3.0 LANGUAGES CXX) +project(fast_float VERSION 6.0.0 LANGUAGES CXX) option(FASTFLOAT_TEST "Enable tests" OFF) if(FASTFLOAT_TEST) enable_testing() diff --git a/README.md b/README.md index 2028051..79ebedb 100644 --- a/README.md +++ b/README.md @@ -371,7 +371,7 @@ the command line help. You may directly download automatically generated single-header files: -https://github.com/fastfloat/fast_float/releases/download/v5.3.0/fast_float.h +https://github.com/fastfloat/fast_float/releases/download/v6.0.0/fast_float.h ## Credit From 7f46adc19c8f3c3034fa9da91718db7b52c18763 Mon Sep 17 00:00:00 2001 From: StefanBruens Date: Tue, 26 Dec 2023 01:56:05 +0100 Subject: [PATCH 27/30] Make tests depending on supplemental_test_files optional As the supplemental_test_files are quite large, it is useful to make running the tests depending on it optional. By default, the tests are kept enabled, but can be switched of by setting `FASTFLOAT_SUPPLEMENTAL_TEST=OFF`. Fixes: #232 --- tests/CMakeLists.txt | 33 +++++++++++++++++++++------------ tests/basictest.cpp | 2 ++ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2cf649c..2718f47 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,16 +5,18 @@ cmake_minimum_required(VERSION 3.11 FATAL_ERROR) include(FetchContent) option(SYSTEM_DOCTEST "Use system copy of doctest" OFF) +option(FASTFLOAT_SUPPLEMENTAL_TESTS "Run supplemental tests" ON) if (NOT SYSTEM_DOCTEST) FetchContent_Declare(doctest GIT_REPOSITORY https://github.com/onqtam/doctest.git GIT_TAG v2.4.10) endif() -FetchContent_Declare(supplemental_test_files - GIT_REPOSITORY https://github.com/fastfloat/supplemental_test_files.git - GIT_TAG origin/main) - +if (FASTFLOAT_SUPPLEMENTAL_TESTS) + FetchContent_Declare(supplemental_test_files + GIT_REPOSITORY https://github.com/fastfloat/supplemental_test_files.git + GIT_TAG origin/main) +endif() # FetchContent_MakeAvailable() was only introduced in 3.14 @@ -27,15 +29,19 @@ if (NOT SYSTEM_DOCTEST) add_subdirectory(${doctest_SOURCE_DIR} ${doctest_BINARY_DIR}) endif() endif() -FetchContent_GetProperties(supplemental_test_files) -if(NOT supplemental_test_files_POPULATED) - message(STATUS "Tests enabled. Retrieving test files.") - FetchContent_Populate(supplemental_test_files) - message(STATUS "Test files retrieved.") - add_subdirectory(${supplemental_test_files_SOURCE_DIR} ${supplemental_test_files_BINARY_DIR}) -endif() + add_library(supplemental-data INTERFACE) -target_compile_definitions(supplemental-data INTERFACE SUPPLEMENTAL_TEST_DATA_DIR="${supplemental_test_files_BINARY_DIR}/data") +if (FASTFLOAT_SUPPLEMENTAL_TESTS) + FetchContent_GetProperties(supplemental_test_files) + if(NOT supplemental_test_files_POPULATED) + message(STATUS "Supplemental tests enabled. Retrieving test files.") + FetchContent_Populate(supplemental_test_files) + message(STATUS "Supplemental test files retrieved.") + add_subdirectory(${supplemental_test_files_SOURCE_DIR} ${supplemental_test_files_BINARY_DIR}) + endif() + target_compile_definitions(supplemental-data INTERFACE SUPPLEMENTAL_TEST_DATA_DIR="${supplemental_test_files_BINARY_DIR}/data") +endif() + function(fast_float_add_cpp_test TEST_NAME) add_executable(${TEST_NAME} ${TEST_NAME}.cpp) if(CMAKE_CROSSCOMPILING) @@ -67,6 +73,9 @@ if (FASTFLOAT_CONSTEXPR_TESTS) else() target_compile_features(basictest PRIVATE cxx_std_17) endif() +if (FASTFLOAT_SUPPLEMENTAL_TESTS) + target_compile_definitions(basictest PRIVATE FASTFLOAT_SUPPLEMENTAL_TESTS) +endif() fast_float_add_cpp_test(long_test) fast_float_add_cpp_test(powersoffive_hardround) diff --git a/tests/basictest.cpp b/tests/basictest.cpp index 3d7f753..b1b6769 100644 --- a/tests/basictest.cpp +++ b/tests/basictest.cpp @@ -246,6 +246,7 @@ TEST_CASE("parse_negative_zero") { CHECK(float64_parsed == 0x8000'0000'0000'0000); } +#if FASTFLOAT_SUPPLEMENTAL_TESTS // C++ 17 because it is otherwise annoying to browse all files in a directory. // We also only run these tests on little endian systems. #if (FASTFLOAT_CPLUSPLUS >= 201703L) && (FASTFLOAT_IS_BIG_ENDIAN == 0) && !defined(FASTFLOAT_ODDPLATFORM) @@ -336,6 +337,7 @@ TEST_CASE("supplemental") { } } #endif +#endif TEST_CASE("leading_zeroes") { constexpr const uint64_t bit = 1; From 5c2a4a026afb260d01e6247e2267b9c3c88b2336 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 26 Dec 2023 16:09:07 -0500 Subject: [PATCH 28/30] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 79ebedb..fbae7a3 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,10 @@ You may directly download automatically generated single-header files: https://github.com/fastfloat/fast_float/releases/download/v6.0.0/fast_float.h +## RFC 7159 + +If you need support for RFC 7159 (JSON standard), you may want to consider using the [fast_double_parser](https://github.com/lemire/fast_double_parser/) library instead. + ## Credit Though this work is inspired by many different people, this work benefited especially from exchanges with From b90ba259efe27a082ee641233c9e447cba997cf2 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Tue, 26 Dec 2023 16:10:05 -0500 Subject: [PATCH 29/30] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fbae7a3..815a65a 100644 --- a/README.md +++ b/README.md @@ -305,7 +305,7 @@ The fast_float library provides a performance similar to that of the [fast_doubl ## Users -The fast_float library is used by [Apache Arrow](https://github.com/apache/arrow/pull/8494) where it multiplied the number parsing speed by two or three times. It is also used by [Yandex ClickHouse](https://github.com/ClickHouse/ClickHouse) and by [Google Jsonnet](https://github.com/google/jsonnet). +The fast_float library is used by [Apache Arrow](https://github.com/apache/arrow/pull/8494) where it multiplied the number parsing speed by two or three times. It is also used by [ClickHouse](https://github.com/ClickHouse/ClickHouse) and by [Google Jsonnet](https://github.com/google/jsonnet). It is part of GCC (as of GCC 12). It is part of WebKit (Safari). ## How fast is it? From b43f8081904d953e1afd01cad7a8e57b6c0ab140 Mon Sep 17 00:00:00 2001 From: Darrell Wright Date: Wed, 27 Dec 2023 19:43:56 -0500 Subject: [PATCH 30/30] Update float_common.h The construct !! is a no-op for a bool, op< for uint64_t's. Removed it and made it an explicit cast to match the operations being performed --- include/fast_float/float_common.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index b5a357b..a7dfac0 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -284,10 +284,10 @@ uint64_t umul128_generic(uint64_t ab, uint64_t cd, uint64_t *hi) { uint64_t ad = emulu((uint32_t)(ab >> 32), (uint32_t)cd); uint64_t bd = emulu((uint32_t)ab, (uint32_t)cd); uint64_t adbc = ad + emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); + uint64_t adbc_carry = (uint64_t)(adbc < ad); uint64_t lo = bd + (adbc << 32); *hi = emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); + (adbc_carry << 32) + (uint64_t)(lo < bd); return lo; }