Remove json parse rules/allow inf_nan

This commit is contained in:
Maya Warrier 2023-05-01 19:45:50 -04:00
parent 091458d192
commit e08c55c380
4 changed files with 25 additions and 77 deletions

2
.gitignore vendored
View File

@ -7,7 +7,7 @@ compile_commands.json
.vs/ .vs/
Debug/ Debug/
Release/ Release/
/out/build/ /out/
*.sln *.sln
*.vcxproj *.vcxproj
*.vcxproj.filters *.vcxproj.filters

View File

@ -51,7 +51,7 @@ fastfloat_really_inline
uint64_t simd_read8_to_u64(const char16_t* chars, const __m128i packus_masks) { uint64_t simd_read8_to_u64(const char16_t* chars, const __m128i packus_masks) {
FASTFLOAT_SIMD_DISABLE_WARNINGS FASTFLOAT_SIMD_DISABLE_WARNINGS
// process 4 and 4 chars simultaneously (loadu_si64 has high latency) // process 4 and 4 chars simultaneously (loadu_si64 has high latency)
// with AVX512BW + AVX512VL, masking is not required as we have cvtepi16_epi8 // with AVX512BW + AVX512VL, masking is not required as we can use cvtepi16_epi8
const char* const p = reinterpret_cast<const char*>(chars); const char* const p = reinterpret_cast<const char*>(chars);
__m128i i1 = _mm_and_si128(_mm_loadu_si64(p), packus_masks); __m128i i1 = _mm_and_si128(_mm_loadu_si64(p), packus_masks);
__m128i i2 = _mm_and_si128(_mm_loadu_si64(p + 8), packus_masks); __m128i i2 = _mm_and_si128(_mm_loadu_si64(p + 8), packus_masks);
@ -223,8 +223,7 @@ template <typename CharT>
fastfloat_really_inline FASTFLOAT_CONSTEXPR20 fastfloat_really_inline FASTFLOAT_CONSTEXPR20
parsed_number_string<CharT> parse_number_string(const CharT *p, const CharT *pend, parse_options options) noexcept { parsed_number_string<CharT> parse_number_string(const CharT *p, const CharT *pend, parse_options options) noexcept {
const chars_format fmt = options.format; const chars_format fmt = options.format;
const parse_rules rules = options.rules; const CharT decimal_point = options.decimal_point;
const CharT decimal_point = CharT(options.decimal_point);
parsed_number_string<CharT> answer; parsed_number_string<CharT> answer;
answer.valid = false; answer.valid = false;
@ -240,7 +239,7 @@ parsed_number_string<CharT> parse_number_string(const CharT *p, const CharT *pen
return answer; return answer;
} }
// a sign must be followed by an integer or the dot // a sign must be followed by an integer or the dot
if (!is_integer(*p) && (rules == parse_rules::json_rules || *p != decimal_point)) if (!is_integer(*p) && *p != decimal_point)
return answer; return answer;
} }
const CharT *const start_digits = p; const CharT *const start_digits = p;
@ -275,8 +274,8 @@ parsed_number_string<CharT> parse_number_string(const CharT *p, const CharT *pen
answer.fraction = span<const CharT>(before, size_t(p - before)); answer.fraction = span<const CharT>(before, size_t(p - before));
digit_count -= exponent; digit_count -= exponent;
} }
// we must have encountered at least one integer (or two if a decimal point exists, with json rules). // we must have encountered at least one integer
if (digit_count == 0 || (rules == parse_rules::json_rules && has_decimal_point && digit_count == 1)) { if (digit_count == 0) {
return answer; return answer;
} }
int64_t exp_number = 0; // explicit exponential part int64_t exp_number = 0; // explicit exponential part
@ -312,11 +311,6 @@ parsed_number_string<CharT> parse_number_string(const CharT *p, const CharT *pen
// If it scientific and not fixed, we have to bail out. // If it scientific and not fixed, we have to bail out.
if((fmt & chars_format::scientific) && !(fmt & chars_format::fixed)) { return answer; } if((fmt & chars_format::scientific) && !(fmt & chars_format::fixed)) { return answer; }
} }
// disallow leading zeros before the decimal point
if (rules == parse_rules::json_rules && start_digits[0] == CharT('0') && digit_count >= 2 && is_integer(start_digits[1]))
return answer;
answer.lastmatch = p; answer.lastmatch = p;
answer.valid = true; answer.valid = true;
answer.exp_number = exp_number; answer.exp_number = exp_number;

View File

@ -13,11 +13,6 @@ enum chars_format {
general = fixed | scientific general = fixed | scientific
}; };
enum parse_rules {
std_rules,
json_rules,
};
template <typename CharT> template <typename CharT>
struct from_chars_result { struct from_chars_result {
const CharT *ptr; const CharT *ptr;
@ -26,34 +21,15 @@ struct from_chars_result {
struct parse_options { struct parse_options {
constexpr explicit parse_options( constexpr explicit parse_options(
chars_format fmt = chars_format::general, chars_format fmt = chars_format::general, char dot = '.')
parse_rules rules = parse_rules::std_rules, : format(fmt), decimal_point(dot) {}
char dot = '.', bool allow_inf_nan = true)
: format(fmt), rules(rules), allow_inf_nan(allow_inf_nan), decimal_point(dot) {}
/** Which number formats are accepted */ /** Which number formats are accepted */
chars_format format; chars_format format;
/** Which parsing rules to use */
parse_rules rules;
/** Whether to allow inf and nan */
bool allow_inf_nan;
/** The character used as decimal point */ /** The character used as decimal point */
char decimal_point; char decimal_point;
}; };
struct preparsed_parse_options {
constexpr explicit preparsed_parse_options(
bool allow_inf_nan = true)
: allow_inf_nan(allow_inf_nan) {}
constexpr preparsed_parse_options(
const parse_options& options)
: allow_inf_nan(options.allow_inf_nan) {}
/** Whether to allow inf and nan */
bool allow_inf_nan;
};
/** /**
* This function parses the character sequence [first,last) for a number. It parses floating-point numbers expecting * This function parses the character sequence [first,last) for a number. It parses floating-point numbers expecting
* a locale-indepent format equivalent to what is used by std::strtod in the default ("C") locale. * a locale-indepent format equivalent to what is used by std::strtod in the default ("C") locale.
@ -88,15 +64,6 @@ from_chars_result<CharT> from_chars_advanced(const CharT *first, const CharT *la
} }
#include "ascii_number.h" // parsed_number_string
namespace fast_float {
template <typename T, typename CharT>
FASTFLOAT_CONSTEXPR20
from_chars_result<CharT> from_chars_preparsed(parsed_number_string<CharT> parsed,
const CharT* first, const CharT* last, T& value, preparsed_parse_options options) noexcept;
}
// namespace fast_float // namespace fast_float
#include "parse_number.h" #include "parse_number.h"
#endif // FASTFLOAT_FAST_FLOAT_H #endif // FASTFLOAT_FAST_FLOAT_H

View File

@ -143,23 +143,30 @@ from_chars_result<CharT> from_chars(const CharT *first, const CharT *last,
template<typename T, typename CharT> template<typename T, typename CharT>
FASTFLOAT_CONSTEXPR20 FASTFLOAT_CONSTEXPR20
from_chars_result<CharT> from_chars_preparsed(parsed_number_string<CharT> pns, const CharT* first, const CharT* last, T& value, preparsed_parse_options options) noexcept from_chars_result<CharT> from_chars_advanced(const CharT *first, const CharT *last,
T &value, parse_options options) noexcept {
{ {
static_assert (std::is_same<T, double>::value || std::is_same<T, float>::value, "only float and double are supported"); static_assert (std::is_same<T, double>::value || std::is_same<T, float>::value, "only float and double are supported");
from_chars_result<CharT> answer; from_chars_result<CharT> answer;
if (!pns.valid) { #ifdef FASTFLOAT_SKIP_WHITE_SPACE // disabled by default
if (options.allow_inf_nan) while ((first != last) && fast_float::is_space(uint8_t(*first))) {
return detail::parse_infnan(first, last, value); first++;
else { }
#endif
if (first == last) {
answer.ec = std::errc::invalid_argument; answer.ec = std::errc::invalid_argument;
answer.ptr = first; answer.ptr = first;
return answer; return answer;
} }
parsed_number_string<CharT> pns = parse_number_string(first, last, options);
if (!pns.valid) {
return detail::parse_infnan(first, last, value);
} }
if (pns.too_many_digits) if (pns.too_many_digits) {
parse_truncated_number_string(pns); parse_truncated_number_string(pns);
}
answer.ec = std::errc(); // be optimistic answer.ec = std::errc(); // be optimistic
answer.ptr = pns.lastmatch; answer.ptr = pns.lastmatch;
@ -220,26 +227,6 @@ from_chars_result<CharT> from_chars_preparsed(parsed_number_string<CharT> pns, c
return answer; return answer;
} }
template<typename T, typename CharT>
FASTFLOAT_CONSTEXPR20
from_chars_result<CharT> from_chars_advanced(const CharT *first, const CharT *last,
T &value, parse_options options) noexcept {
from_chars_result<CharT> answer;
#ifdef FASTFLOAT_SKIP_WHITE_SPACE // disabled by default
while ((first != last) && fast_float::is_space(uint8_t(*first))) {
first++;
}
#endif
if (first == last) {
answer.ec = std::errc::invalid_argument;
answer.ptr = first;
return answer;
}
answer = from_chars_preparsed(parse_number_string(first, last, options), first, last, value, options);
return answer;
}
} // namespace fast_float } // namespace fast_float
#endif #endif