Merge pull request #38 from lemire/dlemire/issue37

Adding support for big endian systems
2025-12-10 02:36:48 +08:00 · 2020-11-16 13:42:54 -05:00 · 2020-11-16 13:42:54 -05:00 · 3957642499
commit 3957642499
parent db539b211c 1b5e3f3945
4 changed files with 226 additions and 1 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -0,0 +1,174 @@
+language: cpp
+
+dist: bionic
+
+arch:
+  - amd64
+  - ppc64le
+  - s390x
+
+cache:
+  directories:
+    - $HOME/.dep_cache
+
+env:
+  global:
+    - fastfloat_DEPENDENCY_CACHE_DIR=$HOME/.dep_cache
+
+matrix:
+  include:
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-8
+      env:
+        - COMPILER="CC=gcc-8 && CXX=g++-8"
+      compiler: gcc-8
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-9
+      env:
+        - COMPILER="CC=gcc-9 && CXX=g++-9"
+      compiler: gcc-9
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-10
+      env:
+        - COMPILER="CC=gcc-10 && CXX=g++-10"
+      compiler: gcc-10
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-10
+      env:
+        - COMPILER="CC=gcc-10 && CXX=g++-10"
+        - SANITIZE="on"
+      compiler: gcc-10-sanitize
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - ubuntu-toolchain-r-test
+          packages:
+            - g++-10
+      env:
+        - COMPILER="CC=gcc-10 && CXX=g++-10"
+        - STATIC="on"
+      compiler: gcc-10-static
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - llvm-toolchain-bionic-6.0
+          packages:
+            - clang-6.0
+      env:
+        - COMPILER="CC=clang-6.0 && CXX=clang++-6.0"
+      compiler: clang-6
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - llvm-toolchain-bionic-7
+          packages:
+            - clang-7
+      env:
+        - COMPILER="CC=clang-7 && CXX=clang++-7"
+      compiler: clang-7
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - llvm-toolchain-bionic-8
+          packages:
+            - clang-8
+      env:
+        - COMPILER="CC=clang-8 && CXX=clang++-8"
+      compiler: clang-8
+
+    - os: linux
+      addons:
+        apt:
+          sources:
+            - llvm-toolchain-bionic-9
+          packages:
+            - clang-9
+      env:
+        - COMPILER="CC=clang-9 && CXX=clang++-9"
+      compiler: clang-9
+
+    - os: linux
+      addons:
+        apt:
+          packages:
+            - clang-10
+          sources:
+            - ubuntu-toolchain-r-test
+            - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main'
+              key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
+      env:
+        - COMPILER="CC=clang-10 && CXX=clang++-10"
+      compiler: clang-10
+
+    - os: linux
+      addons:
+        apt:
+          packages:
+            - clang-10
+          sources:
+            - ubuntu-toolchain-r-test
+            - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main'
+              key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
+      env:
+        - COMPILER="CC=clang-10 && CXX=clang++-10"
+        - STATIC="on"
+      compiler: clang-10-static
+
+    - os: linux
+      addons:
+        apt:
+          packages:
+            - clang-10
+          sources:
+            - ubuntu-toolchain-r-test
+            - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main'
+              key_url: 'https://apt.llvm.org/llvm-snapshot.gpg.key'
+      env:
+        - COMPILER="CC=clang-10 && CXX=clang++-10"
+        - SANITIZE="on"
+      compiler: clang-10-sanitize
+
+before_install:
+  - eval "${COMPILER}"
+
+install:
+  - sudo apt-get -qq update
+
+script:
+  - mkdir build
+  - cd build
+  - cmake  -DFASTFLOAT_TEST=ON  ..
+  - make
+  - ctest --output-on-failure -R basictest
+
--- a/include/fast_float/ascii_number.h
+++ b/include/fast_float/ascii_number.h
@ -78,6 +78,8 @@ parsed_number_string parse_number_string(const char *p, const char *pend, chars_
  if ((p != pend) && (*p == '.')) {
    ++p;
    const char *first_after_period = p;
+#if FASTFLOAT_IS_BIG_ENDIAN == 0
+    // Fast approach only tested under little endian systems
    if ((p + 8 <= pend) && is_made_of_eight_digits_fast(p)) {
      i = i * 100000000 + parse_eight_digits_unrolled(p); // in rare cases, this will overflow, but that's ok
      p += 8;
@ -86,6 +88,7 @@ parsed_number_string parse_number_string(const char *p, const char *pend, chars_
        p += 8;
      }
    }
+#endif
    while ((p != pend) && is_integer(*p)) {
      uint8_t digit = uint8_t(*p - '0');
      ++p;
@ -196,6 +199,7 @@ fastfloat_really_inline decimal parse_decimal(const char *p, const char *pend) n
       ++p;
      }
    }
+#if FASTFLOAT_IS_BIG_ENDIAN == 0
    // We expect that this loop will often take the bulk of the running time
    // because when a value has lots of digits, these digits often
    while ((p + 8 <= pend) && (answer.num_digits + 8 < max_digits)) {
@ -208,6 +212,7 @@ fastfloat_really_inline decimal parse_decimal(const char *p, const char *pend) n
      answer.num_digits += 8;
      p += 8;
    }
+#endif
    while ((p != pend) && is_integer(*p)) {
      if (answer.num_digits < max_digits) {
        answer.digits[answer.num_digits] = uint8_t(*p - '0');
--- a/include/fast_float/float_common.h
+++ b/include/fast_float/float_common.h
@ -8,6 +8,34 @@
 #define FASTFLOAT_VISUAL_STUDIO 1
 #endif

+
+
+#ifdef _WIN32
+#define FASTFLOAT_IS_BIG_ENDIAN 0
+#else
+#if defined(__APPLE__) || defined(__FreeBSD__)
+#include <machine/endian.h>
+#else
+#include <endian.h>
+#endif
+#
+#ifndef __BYTE_ORDER__
+// safe choice
+#define FASTFLOAT_IS_BIG_ENDIAN 0
+#endif
+#
+#ifndef __ORDER_LITTLE_ENDIAN__
+// safe choice
+#define FASTFLOAT_IS_BIG_ENDIAN 0
+#endif
+#
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+#define FASTFLOAT_IS_BIG_ENDIAN 0
+#else
+#define FASTFLOAT_IS_BIG_ENDIAN 1
+#endif
+#endif
+
 #ifdef FASTFLOAT_VISUAL_STUDIO
 #define fastfloat_really_inline __forceinline
 #else
@ -154,6 +182,14 @@ struct decimal {
  // Note that the user is responsible to ensure that digits are
  // initialized to zero when there are fewer than 19.
  inline uint64_t to_truncated_mantissa() {
+#if FASTFLOAT_IS_BIG_ENDIAN == 1
+    uint64_t mantissa = 0;
+    for (uint32_t i = 0; i < max_digit_without_overflow;
+         i++) {
+      mantissa = mantissa * 10 + digits[i]; // can be accelerated
+    }
+    return mantissa;
+#else
    uint64_t val;
    // 8 first digits
    ::memcpy(&val, digits, sizeof(uint64_t));
@ -173,6 +209,7 @@ struct decimal {
      mantissa = mantissa * 10 + digits[i]; // can be accelerated
    }
    return mantissa;
+#endif
  }
  // Generate san exponent matching to_truncated_mantissa()
  inline int32_t to_truncated_exponent() {
--- a/include/fast_float/parse_number.h
+++ b/include/fast_float/parse_number.h
@ -107,7 +107,16 @@ from_chars_result from_chars(const char *first, const char *last,
  word |= uint64_t(am.power2) << binary_format<T>::mantissa_explicit_bits();
  word = pns.negative
  ? word | (uint64_t(1) << binary_format<T>::sign_index()) : word;
-  ::memcpy(&value, &word, sizeof(T));
+#if FASTFLOAT_IS_BIG_ENDIAN == 1
+   if (std::is_same<T, float>::value) {
+     ::memcpy(&value, (char *)&word + 4, sizeof(T)); // extract value at offset 4-7 if float on big-endian
+   } else {
+     ::memcpy(&value, &word, sizeof(T));
+   }
+#else
+   // For little-endian systems:
+   ::memcpy(&value, &word, sizeof(T));
+#endif
  return answer;
 }