From 7ff364b59a9900a21f26c2609b2037277117b5eb Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 16 Nov 2020 12:04:57 -0500 Subject: [PATCH 1/5] This might add support for big endian systems (untested). --- include/fast_float/ascii_number.h | 5 +++++ include/fast_float/float_common.h | 36 +++++++++++++++++++++++++++++++ include/fast_float/parse_number.h | 11 +++++++++- 3 files changed, 51 insertions(+), 1 deletion(-) diff --git a/include/fast_float/ascii_number.h b/include/fast_float/ascii_number.h index bd072c6..d39ad52 100644 --- a/include/fast_float/ascii_number.h +++ b/include/fast_float/ascii_number.h @@ -78,6 +78,8 @@ parsed_number_string parse_number_string(const char *p, const char *pend, chars_ if ((p != pend) && (*p == '.')) { ++p; const char *first_after_period = p; +#if FASTFLOAT_IS_BIG_ENDIAN == 0 + // Fast approach only tested under little endian systems if ((p + 8 <= pend) && is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); // in rare cases, this will overflow, but that's ok p += 8; @@ -86,6 +88,7 @@ parsed_number_string parse_number_string(const char *p, const char *pend, chars_ p += 8; } } +#endif while ((p != pend) && is_integer(*p)) { uint8_t digit = uint8_t(*p - '0'); ++p; @@ -196,6 +199,7 @@ fastfloat_really_inline decimal parse_decimal(const char *p, const char *pend) n ++p; } } +#if FASTFLOAT_IS_BIG_ENDIAN == 0 // We expect that this loop will often take the bulk of the running time // because when a value has lots of digits, these digits often while ((p + 8 <= pend) && (answer.num_digits + 8 < max_digits)) { @@ -208,6 +212,7 @@ fastfloat_really_inline decimal parse_decimal(const char *p, const char *pend) n answer.num_digits += 8; p += 8; } +#endif while ((p != pend) && is_integer(*p)) { if (answer.num_digits < max_digits) { answer.digits[answer.num_digits] = uint8_t(*p - '0'); diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index 7879654..a208741 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -8,6 +8,34 @@ #define FASTFLOAT_VISUAL_STUDIO 1 #endif + + +#ifdef _WIN32 +#define FASTFLOAT_IS_BIG_ENDIAN 0 +#else +#if defined(__APPLE__) || defined(__FreeBSD__) +#include +#else +#include +#endif +# +#ifndef __BYTE_ORDER__ +// safe choice +#define FASTFLOAT_IS_BIG_ENDIAN 0 +#endif +# +#ifndef __ORDER_LITTLE_ENDIAN__ +// safe choice +#define FASTFLOAT_IS_BIG_ENDIAN 0 +#endif +# +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define FASTFLOAT_IS_BIG_ENDIAN 0 +#else +#define FASTFLOAT_IS_BIG_ENDIAN 1 +#endif +#endif + #ifdef FASTFLOAT_VISUAL_STUDIO #define fastfloat_really_inline __forceinline #else @@ -154,6 +182,13 @@ struct decimal { // Note that the user is responsible to ensure that digits are // initialized to zero when there are fewer than 19. inline uint64_t to_truncated_mantissa() { +#if FASTFLOAT_IS_BIG_ENDIAN == 1 + for (uint32_t i = 0; i < max_digit_without_overflow; + i++) { + mantissa = mantissa * 10 + digits[i]; // can be accelerated + } + return mantissa; +#else uint64_t val; // 8 first digits ::memcpy(&val, digits, sizeof(uint64_t)); @@ -173,6 +208,7 @@ struct decimal { mantissa = mantissa * 10 + digits[i]; // can be accelerated } return mantissa; +#endif } // Generate san exponent matching to_truncated_mantissa() inline int32_t to_truncated_exponent() { diff --git a/include/fast_float/parse_number.h b/include/fast_float/parse_number.h index 4652cc8..1b7a419 100644 --- a/include/fast_float/parse_number.h +++ b/include/fast_float/parse_number.h @@ -107,7 +107,16 @@ from_chars_result from_chars(const char *first, const char *last, word |= uint64_t(am.power2) << binary_format::mantissa_explicit_bits(); word = pns.negative ? word | (uint64_t(1) << binary_format::sign_index()) : word; - ::memcpy(&value, &word, sizeof(T)); +#if FASTFLOAT_IS_BIG_ENDIAN == 1 + if (std::is_same::value) { + ::memcpy(&value, (char *)&word + 4, sizeof(T)); // extract value at offset 4-7 if float on big-endian + } else { + ::memcpy(&value, &word, sizeof(T)); + } +#else + // For little-endian systems: + ::memcpy(&value, &word, sizeof(T)); +#endif return answer; } From 1d9116b4f241eec3286b02143305f6719b8658d8 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 16 Nov 2020 12:17:20 -0500 Subject: [PATCH 2/5] Adding travis. --- .travis.yml | 188 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 188 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..29746eb --- /dev/null +++ b/.travis.yml @@ -0,0 +1,188 @@ +language: cpp + +dist: bionic + +arch: + - amd64 + - ppc64le + - s390x + +cache: + directories: + - $HOME/.dep_cache + +env: + global: + - simdjson_DEPENDENCY_CACHE_DIR=$HOME/.dep_cache + +matrix: + include: + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-8 + env: + - COMPILER="CC=gcc-8 && CXX=g++-8" + compiler: gcc-8 + + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-9 + env: + - COMPILER="CC=gcc-9 && CXX=g++-9" + compiler: gcc-9 + + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-10 + env: + - COMPILER="CC=gcc-10 && CXX=g++-10" + compiler: gcc-10 + + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-10 + env: + - COMPILER="CC=gcc-10 && CXX=g++-10" + - SANITIZE="on" + compiler: gcc-10-sanitize + + - os: linux + addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - g++-10 + env: + - COMPILER="CC=gcc-10 && CXX=g++-10" + - STATIC="on" + compiler: gcc-10-static + + - os: linux + addons: + apt: + sources: + - llvm-toolchain-bionic-6.0 + packages: + - clang-6.0 + env: + - COMPILER="CC=clang-6.0 && CXX=clang++-6.0" + compiler: clang-6 + + - os: linux + addons: + apt: + sources: + - llvm-toolchain-bionic-7 + packages: + - clang-7 + env: + - COMPILER="CC=clang-7 && CXX=clang++-7" + compiler: clang-7 + + - os: linux + addons: + apt: + sources: + - llvm-toolchain-bionic-8 + packages: + - clang-8 + env: + - COMPILER="CC=clang-8 && CXX=clang++-8" + compiler: clang-8 + + - os: linux + addons: + apt: + sources: + - llvm-toolchain-bionic-9 + packages: + - clang-9 + env: + - COMPILER="CC=clang-9 && CXX=clang++-9" + compiler: clang-9 + + - os: linux + addons: + apt: + packages: + - clang-10 + sources: + - ubuntu-toolchain-r-test + - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + env: + - COMPILER="CC=clang-10 && CXX=clang++-10" + compiler: clang-10 + + - os: linux + addons: + apt: + packages: + - clang-10 + sources: + - ubuntu-toolchain-r-test + - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + env: + - COMPILER="CC=clang-10 && CXX=clang++-10" + - STATIC="on" + compiler: clang-10-static + + - os: linux + addons: + apt: + packages: + - clang-10 + sources: + - ubuntu-toolchain-r-test + - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main' + key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key' + env: + - COMPILER="CC=clang-10 && CXX=clang++-10" + - SANITIZE="on" + compiler: clang-10-sanitize + +before_install: + - eval "${COMPILER}" + +install: + - wget -q -O - "https://raw.githubusercontent.com/simdjson/debian-ppa/master/key.gpg" | sudo apt-key add - + - sudo apt-add-repository "deb https://raw.githubusercontent.com/simdjson/debian-ppa/master simdjson main" + - sudo apt-get -qq update + - sudo apt-get purge cmake cmake-data + - sudo apt-get -t simdjson -y install cmake + - export CMAKE_CXX_FLAGS="-maltivec -mcpu=power9 -mtune=power9" + - export CMAKE_C_FLAGS="${CMAKE_CXX_FLAGS}" + - export CMAKE_FLAGS="-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -DSIMDJSON_IMPLEMENTATION=ppc64;fallback"; + - if [[ "${SANITIZE}" == "on" ]]; then + export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_SANITIZE=ON"; + export ASAN_OPTIONS="detect_leaks=0"; + fi + - if [[ "${STATIC}" == "on" ]]; then + export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_BUILD_STATIC=ON"; + fi + - export CTEST_FLAGS="-j4 --output-on-failure -E checkperf" + +script: + - cmake -DFASTFLOAT_TEST=ON -B build + - cmake --build build + - cd build + - ctest --output-on-failure -R basictest + From 59d5313432e7c6c6dccfa9e16e3353621f29b933 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 16 Nov 2020 12:36:15 -0500 Subject: [PATCH 3/5] Trimming. --- .travis.yml | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 29746eb..067abe0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ cache: env: global: - - simdjson_DEPENDENCY_CACHE_DIR=$HOME/.dep_cache + - fastfloat_DEPENDENCY_CACHE_DIR=$HOME/.dep_cache matrix: include: @@ -163,22 +163,7 @@ before_install: - eval "${COMPILER}" install: - - wget -q -O - "https://raw.githubusercontent.com/simdjson/debian-ppa/master/key.gpg" | sudo apt-key add - - - sudo apt-add-repository "deb https://raw.githubusercontent.com/simdjson/debian-ppa/master simdjson main" - sudo apt-get -qq update - - sudo apt-get purge cmake cmake-data - - sudo apt-get -t simdjson -y install cmake - - export CMAKE_CXX_FLAGS="-maltivec -mcpu=power9 -mtune=power9" - - export CMAKE_C_FLAGS="${CMAKE_CXX_FLAGS}" - - export CMAKE_FLAGS="-DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -DSIMDJSON_IMPLEMENTATION=ppc64;fallback"; - - if [[ "${SANITIZE}" == "on" ]]; then - export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_SANITIZE=ON"; - export ASAN_OPTIONS="detect_leaks=0"; - fi - - if [[ "${STATIC}" == "on" ]]; then - export CMAKE_FLAGS="${CMAKE_FLAGS} -DSIMDJSON_BUILD_STATIC=ON"; - fi - - export CTEST_FLAGS="-j4 --output-on-failure -E checkperf" script: - cmake -DFASTFLOAT_TEST=ON -B build From 7ebbb9d5be9a2a3c14f310b7d6b30f9e8f7a2f1e Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 16 Nov 2020 12:40:05 -0500 Subject: [PATCH 4/5] Going old school. --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 067abe0..616ab64 100644 --- a/.travis.yml +++ b/.travis.yml @@ -166,8 +166,9 @@ install: - sudo apt-get -qq update script: - - cmake -DFASTFLOAT_TEST=ON -B build - - cmake --build build + - mkdir build - cd build + - cmake -DFASTFLOAT_TEST=ON .. + - make - ctest --output-on-failure -R basictest From 1b5e3f3945a5e2839252675d79728a5325023e19 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Mon, 16 Nov 2020 12:56:57 -0500 Subject: [PATCH 5/5] patching be support. (typo) --- include/fast_float/float_common.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/fast_float/float_common.h b/include/fast_float/float_common.h index a208741..60e5f6e 100644 --- a/include/fast_float/float_common.h +++ b/include/fast_float/float_common.h @@ -183,6 +183,7 @@ struct decimal { // initialized to zero when there are fewer than 19. inline uint64_t to_truncated_mantissa() { #if FASTFLOAT_IS_BIG_ENDIAN == 1 + uint64_t mantissa = 0; for (uint32_t i = 0; i < max_digit_without_overflow; i++) { mantissa = mantissa * 10 + digits[i]; // can be accelerated