From f5f115a3aced999f6a23cedc729b9f3323b9ed65 Mon Sep 17 00:00:00 2001 From: "Node.js GitHub Bot" Date: Mon, 5 Oct 2026 15:20:04 +0000 Subject: [PATCH 1/2] deps: update simdjson to 5.0.2 --- deps/simdjson/simdjson.cpp | 22966 +++++++++-- deps/simdjson/simdjson.h | 73104 +++++++++++++++++++++++++++++++---- 2 files changed, 84550 insertions(+), 11520 deletions(-) diff --git a/deps/simdjson/simdjson.cpp b/deps/simdjson/simdjson.cpp index 71f443b3d2bb..81cd2faf6dff 100644 --- a/deps/simdjson/simdjson.cpp +++ b/deps/simdjson/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2026-09-04 16:04:31 -0400. version 4.6.11 Do not edit! */ +/* auto-generated on 2026-10-04 09:02:57 -0400. version 5.0.2 Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP @@ -41,7 +41,9 @@ #endif // C++ 26 -#if !defined(SIMDJSON_CPLUSPLUS26) && (SIMDJSON_CPLUSPLUS >= 202402L) // update when the standard is finalized +// While C++26 is a working draft, compilers report 202400L in C++26 mode +// (both GCC 16 and Clang 21 do). Update when the standard is finalized. +#if !defined(SIMDJSON_CPLUSPLUS26) && (SIMDJSON_CPLUSPLUS >= 202400L) #define SIMDJSON_CPLUSPLUS26 1 #endif @@ -98,14 +100,48 @@ #endif #endif -// The current specification is unclear on how we detect -// static reflection, both __cpp_lib_reflection and -// __cpp_impl_reflection are proposed in the draft specification. -// For now, we disable static reflect by default. It must be -// specified at compiler time. +// Static reflection. +// +// The reflection-based APIs (simdjson::to, document::get, the builder, +// compile-time JSON, annotations) need considerably more than the reflection +// operator. We turn them on only when the compiler advertises all of: +// +// P2996 reflection (^^, splicers, ) __cpp_impl_reflection, +// __cpp_lib_reflection +// P1306 expansion statements (template for) __cpp_expansion_statements +// P3491 std::define_static_string / _array __cpp_lib_define_static +// +// Two further features we rely on have, as of this writing, no feature-test +// macro of their own, so they cannot be checked directly: +// +// P3394 annotations ([[=x]], std::meta::annotations_of) -- used for +// the annotations of simdjson/annotations.h (rename, skip, ...). +// P3289 consteval blocks (consteval { ... }) -- used by compile_time_json. +// +// Every implementation that defines the four macros above also implements +// those two, so requiring the four is sufficient in practice. If that ever +// stops being true, define SIMDJSON_STATIC_REFLECTION=0 to opt out. +// +// SIMDJSON_STATIC_REFLECTION may always be defined by the user (or by the +// build system) to 0 or 1 to override the detection. +// +// Note that C++26 mode alone is not enough: GCC 16 requires -freflection, +// and only then does it define __cpp_impl_reflection. #ifndef SIMDJSON_STATIC_REFLECTION -#define SIMDJSON_STATIC_REFLECTION 0 // disabled by default. +#if defined(SIMDJSON_CPLUSPLUS26) && \ + defined(__cpp_impl_reflection) && __cpp_impl_reflection >= 202506L && \ + defined(__cpp_lib_reflection) && __cpp_lib_reflection >= 202506L && \ + defined(__cpp_expansion_statements) && \ + __cpp_expansion_statements >= 202506L && \ + defined(__cpp_lib_define_static) && __cpp_lib_define_static >= 202506L +// __cpp_lib_reflection is the feature-test macro for , so there is no +// need for a separate __has_include check (which would have to be guarded for +// compilers that lack __has_include). +#define SIMDJSON_STATIC_REFLECTION 1 +#else +#define SIMDJSON_STATIC_REFLECTION 0 #endif +#endif // SIMDJSON_STATIC_REFLECTION #if defined(__apple_build_version__) #if __apple_build_version__ < 14000000 @@ -138,6 +174,47 @@ #define SIMDJSON_SUPPORTS_DESERIALIZATION 0 #endif +// The C++20 char8_t type (and std::u8string/std::u8string_view) is available. +// Because all strings that simdjson produces are valid UTF-8, we can offer +// char8_t variants of our string accessors when this macro is set. +#if !defined(SIMDJSON_SUPPORTS_CHAR8_T) +#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L +#define SIMDJSON_SUPPORTS_CHAR8_T 1 +#else +#define SIMDJSON_SUPPORTS_CHAR8_T 0 +#endif +#endif // !defined(SIMDJSON_SUPPORTS_CHAR8_T) + +// The C++23 fixed-width floating-point types std::float32_t and std::float64_t +// () are available. They are optional even in C++23: a compiler that +// provides them predefines __STDCPP_FLOAT32_T__ and __STDCPP_FLOAT64_T__. +// When these macros are set, we offer get_float32() and get_float64(). +#if !defined(SIMDJSON_SUPPORTS_FLOAT32_T) +#if defined(__STDCPP_FLOAT32_T__) && defined(__has_include) +#if __has_include() +#define SIMDJSON_SUPPORTS_FLOAT32_T 1 +#endif +#endif +#ifndef SIMDJSON_SUPPORTS_FLOAT32_T +#define SIMDJSON_SUPPORTS_FLOAT32_T 0 +#endif +#endif // !defined(SIMDJSON_SUPPORTS_FLOAT32_T) + +#if !defined(SIMDJSON_SUPPORTS_FLOAT64_T) +#if defined(__STDCPP_FLOAT64_T__) && defined(__has_include) +#if __has_include() +#define SIMDJSON_SUPPORTS_FLOAT64_T 1 +#endif +#endif +#ifndef SIMDJSON_SUPPORTS_FLOAT64_T +#define SIMDJSON_SUPPORTS_FLOAT64_T 0 +#endif +#endif // !defined(SIMDJSON_SUPPORTS_FLOAT64_T) + +#if SIMDJSON_SUPPORTS_FLOAT32_T || SIMDJSON_SUPPORTS_FLOAT64_T +#include +#endif + #if !defined(SIMDJSON_CONSTEVAL) #if defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L @@ -146,6 +223,18 @@ #define SIMDJSON_CONSTEVAL 0 #endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L #endif // !defined(SIMDJSON_CONSTEVAL) + +// SIMDJSON_CONSTEXPR_STRING is 'constexpr' when the standard library supports +// constexpr std::string (e.g., libstdc++ 12 or better), and empty otherwise. It +// lets functions that build a std::string be constant expressions when possible +// while still compiling against older standard libraries. +#if !defined(SIMDJSON_CONSTEXPR_STRING) +#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L +#define SIMDJSON_CONSTEXPR_STRING constexpr +#else +#define SIMDJSON_CONSTEXPR_STRING +#endif // defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L +#endif // !defined(SIMDJSON_CONSTEXPR_STRING) #endif // SIMDJSON_COMPILER_CHECK_H /* end file simdjson/compiler_check.h */ /* including simdjson/portability.h: #include "simdjson/portability.h" */ @@ -437,16 +526,86 @@ using std::size_t; #endif #endif +#ifndef SIMDJSON_HAS_UNISTD_H +#if defined(__unix__) || defined(__APPLE__) || defined(__linux__) +#define SIMDJSON_HAS_UNISTD_H 1 +#else +#define SIMDJSON_HAS_UNISTD_H 0 +#endif +#endif + +// padded_memory_map availability. +// +// On POSIX platforms the class is always available: the implementation uses +// `mmap` (and a trailing anonymous page for padding) from . +// +// On Windows the class is disabled by default and must be explicitly +// opted into by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. Enabling +// it requires: +// 1. `` has been included *before* `` (so that +// this header can see the Win32 types and the `_WINDOWS_` include +// guard), +// 2. the compilation targets Windows 10, version 1803 or later +// (i.e. `NTDDI_VERSION >= NTDDI_WIN10_RS4`, `0x0A000005`). This is +// required because the implementation relies on the modern memory +// APIs introduced with that version (`CreateFileMapping2` / +// `MapViewOfFile3`), +// 3. the link step pulls in an import library that exports those APIs, +// typically `onecore.lib` (or `mincore.lib`). +// +// The `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS` CMake option arranges (1)-(3) +// automatically when building simdjson with its own CMake. Consumers using +// simdjson as a pre-built library are responsible for setting the macro, +// the Windows version macros, and the link library themselves. +// +// If the opt-in conditions are not met on Windows, `padded_memory_map` +// simply does not exist -- any attempt to use it fails at compile time +// with an "unknown identifier" diagnostic rather than silently degrading. +// +// The SIMDJSON_HAS_PADDED_MEMORY_MAP macro reflects whether the class is +// available in the current translation unit. Users may test this macro to +// conditionally compile code that depends on padded_memory_map. +#ifndef SIMDJSON_HAS_PADDED_MEMORY_MAP + #if defined(__unix__) || defined(__APPLE__) || defined(__linux__) + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 + #elif defined(_WINDOWS_) && defined(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS) && SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 + #else + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 0 + #endif +#endif #endif // SIMDJSON_PORTABILITY_H /* end file simdjson/portability.h */ +#include namespace simdjson { namespace internal { +/** + * @private + * Scratch capacity that every caller of to_chars must provide. + * + * The emitted decimal is at most ~24 characters, but dragonbox() and + * format_buffer() intentionally write past the logical end with fixed-size + * 16/17-byte memcpy/memset operations so the compiler can inline them (no + * libc mem* dispatch with size-class branches). The extra bytes are required + * for safety of those over-writes; do not shrink this below 40. + * See src/to_chars.cpp and #2805. + */ +// Use an unscoped enum (not static constexpr / inline constexpr): +// - C++11 targets (readme_examples11, quickstart11, ...) still include this header +// - a static constexpr in the amalgamated simdjson.cpp TU is unused there +// (only callers in headers use it) and trips -Wunused-const-variable -Werror +enum : size_t { to_chars_buffer_size = 40 }; /** * @private * Our own implementation of the C++17 to_chars function. * Defined in src/to_chars + * + * @note The buffer starting at first must have at least to_chars_buffer_size + * bytes of writable storage (see to_chars_buffer_size). + * @note The input number must be finite (NaN/Inf are not supported). + * @note The result is NOT null-terminated. */ char *to_chars(char *first, const char *last, double value); /** @@ -456,6 +615,12 @@ char *to_chars(char *first, const char *last, double value); */ double from_chars(const char *first) noexcept; double from_chars(const char *first, const char* end) noexcept; +/** + * @private + * Same as from_chars, but produces a correctly rounded binary32 (float) value. + * Defined in src/from_chars + */ +float from_chars_float(const char *first) noexcept; } #ifndef SIMDJSON_EXCEPTIONS @@ -466,6 +631,10 @@ double from_chars(const char *first, const char* end) noexcept; #endif #endif +#ifndef SIMDJSON_ENABLE_NAN_INF +#define SIMDJSON_ENABLE_NAN_INF 0 +#endif + } // namespace simdjson #if defined(__GNUC__) @@ -481,16 +650,14 @@ double from_chars(const char *first, const char* end) noexcept; // Align to N-byte boundary #define SIMDJSON_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) -#define SIMDJSON_ROUNDDOWN_N(a, n) ((a) & ~((n)-1)) - -#define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) #if SIMDJSON_REGULAR_VISUAL_STUDIO // We could use [[deprecated]] but it requires C++14 #define simdjson_deprecated __declspec(deprecated) #define simdjson_really_inline __forceinline - #define simdjson_never_inline __declspec(noinline) + #define simdjson_never_inline inline __declspec(noinline) + #define simdjson_really_flatten [[msvc::flatten]] #define simdjson_unused #define simdjson_warn_unused @@ -531,6 +698,7 @@ double from_chars(const char *first, const char* end) noexcept; #define simdjson_really_inline inline __attribute__((always_inline)) #define simdjson_never_inline inline __attribute__((noinline)) + #define simdjson_really_flatten [[gnu::flatten]] #define simdjson_unused __attribute__((unused)) #define simdjson_warn_unused __attribute__((warn_unused_result)) @@ -607,6 +775,15 @@ double from_chars(const char *first, const char* end) noexcept; #define simdjson_inline simdjson_really_inline #endif +#if defined(simdjson_flatten) + // Prefer the user's definition of simdjson_flatten; don't define it ourselves. +#elif (defined(__GNUC__) && !defined(__OPTIMIZE__)) || (defined(_DEBUG) && _MSC_VER ) + // Flattening can lead to significant code bloat and high compile times. Don't use it for unoptimized builds. + #define simdjson_flatten +#else + #define simdjson_flatten simdjson_really_flatten +#endif + #if SIMDJSON_VISUAL_STUDIO /** * Windows users need to do some extra work when building @@ -2562,6 +2739,7 @@ enum error_code { OUT_OF_BOUNDS, ///< Attempted to access location outside of document. TRAILING_CONTENT, ///< Unexpected trailing content in the JSON input OUT_OF_CAPACITY, ///< The capacity was exceeded, we cannot allocate enough memory. + UNKNOWN_FIELD, ///< JSON field does not map to any member of the target (see simdjson::deny_unknown_fields) NUM_ERROR_CODES ///< Placeholder for end of error code list. }; @@ -2924,6 +3102,7 @@ inline const std::string error_message(int error) noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS #include +#include #include namespace simdjson { @@ -2965,6 +3144,19 @@ concept constructible_from_string_view = std::is_constructible_v && std::is_default_constructible_v; +#if SIMDJSON_SUPPORTS_CHAR8_T +/** + * A C++20 char8_t string type such as std::u8string. Such types cannot be built + * from a std::string_view (the character types differ), so they need their own + * deserialization path, going through the u8 string accessors. + */ +template +concept constructible_from_u8string_view = std::is_constructible_v + && !std::is_same_v + && !std::is_constructible_v + && std::is_default_constructible_v; +#endif // SIMDJSON_SUPPORTS_CHAR8_T + template concept string_view_keyed_map = string_view_like && requires(std::remove_cvref_t& m, typename M::key_type sv, typename M::mapped_type v) { @@ -3059,9 +3251,15 @@ concept string_like = // Concept that checks if a type is a container but not a string (because // strings handling must be handled differently) // Now uses iterator-based approach for broader container support +// +// Optional types are excluded on purpose. Since C++26 (P3168), std::optional +// is itself a range, so without the exclusion an std::optional would match +// both this concept and optional_type, making the container and the optional +// overloads of atom()/append() ambiguous. See issue 2827. template concept container_but_not_string = - std::ranges::input_range && !string_like && !concepts::string_view_keyed_map; + std::ranges::input_range && !string_like && !concepts::string_view_keyed_map + && !concepts::optional_type; @@ -3168,6 +3366,11 @@ struct fixed_string { data[i] = str[i]; } } + constexpr fixed_string(const unsigned char (&str)[N]) { + for (std::size_t i = 0; i < N; ++i) { + data[i] = static_cast(str[i]); + } + } char data[N]; constexpr std::string_view view() const { return {data, N - 1}; } constexpr size_t size() const { return N ; } @@ -3199,6 +3402,11 @@ struct string_constant { #endif // SIMDJSON_CONSTEVALUTIL_H /* end file simdjson/constevalutil.h */ +#if SIMDJSON_SUPPORTS_CHAR8_T +#include +#include +#endif + /** * @brief The top level simdjson namespace, containing everything the library provides. */ @@ -3208,6 +3416,10 @@ SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS /** The maximum document size supported by simdjson. */ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF; +/** The maximum depth of nested objects and arrays supported by simdjson. + A depth of SIMDJSON_MAXSIZE_BYTES/2 is not reasonable and would be + adversarial, but it serves as an upper bound for validation purposes. */ +constexpr size_t SIMDJSON_MAX_DEPTH = SIMDJSON_MAXSIZE_BYTES/2; /** * The amount of padding needed in a buffer to parse JSON. @@ -3233,6 +3445,30 @@ struct padded_string; class padded_string_view; enum class stage1_mode; +/** + * Stream format for parse_many/iterate_many. + */ +enum class stream_format { + whitespace_delimited, ///< Whitespace-delimited JSON documents (default, includes NDJSON/JSONL) + json_sequence, ///< RFC 7464 JSON text sequences (RS-delimited) + comma_delimited, ///< Comma-separated JSON documents (e.g., `{...},{...},{...}`) + comma_delimited_array,///< A single JSON array whose elements are iterated as + ///< comma-separated documents (e.g., `[{...},{...},{...}]`). + ///< The parser strips the outer `[` / `]` plus any + ///< surrounding JSON whitespace (space, tab, LF, CR) + ///< and then behaves like `comma_delimited` over the + ///< remaining bytes. + newline_delimited ///< NDJSON/JSON Lines where each document occupies exactly + ///< one line: documents are separated by line feeds and no + ///< document contains a raw line feed. Same inputs as + ///< `whitespace_delimited`, but the stronger guarantee lets + ///< the parser find the end of a document without walking + ///< it. On ondemand `iterate_many`, an unread remainder may + ///< be skipped by jumping to the next line feed without + ///< structure-validating that remainder. Use + ///< `whitespace_delimited` if unsure. +}; + namespace internal { template @@ -3243,6 +3479,52 @@ class tape_ref; struct value128; enum class tape_type; +#if SIMDJSON_SUPPORTS_CHAR8_T +/** + * Reinterpret a UTF-8 string as a C++20 std::u8string_view. No byte is copied + * or modified: char8_t and char have the same size, representation and + * alignment. Every string that simdjson produces is valid UTF-8, so this is a + * lossless view over the very same memory. + * @private + */ +simdjson_inline std::u8string_view as_u8string_view(std::string_view v) noexcept { + return std::u8string_view(reinterpret_cast(v.data()), v.size()); +} +#endif // SIMDJSON_SUPPORTS_CHAR8_T + +/** + * Assign a UTF-8 string to a string-like receiver. The general case simply + * assigns the std::string_view: it covers std::string and any user type that + * can be assigned from a std::string_view. + * @private + */ +template +simdjson_inline void assign_utf8(string_type &receiver, std::string_view content) noexcept { + receiver = content; +} + +#if SIMDJSON_SUPPORTS_CHAR8_T +/** + * Assign a UTF-8 string to a char8_t-based string (e.g., std::u8string). This + * overload is more specialized than the general one, so overload resolution + * prefers it whenever the receiver holds char8_t. + * @private + */ +template +simdjson_inline void assign_utf8(std::basic_string &receiver, std::string_view content) noexcept { + receiver.assign(reinterpret_cast(content.data()), content.size()); +} + +/** + * Assign a UTF-8 string to a char8_t-based string view (e.g., std::u8string_view). + * @private + */ +template +simdjson_inline void assign_utf8(std::basic_string_view &receiver, std::string_view content) noexcept { + receiver = std::basic_string_view(reinterpret_cast(content.data()), content.size()); +} +#endif // SIMDJSON_SUPPORTS_CHAR8_T + } // namespace internal } // namespace simdjson @@ -3263,806 +3545,1038 @@ SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS #include #include -#include #include +#include namespace simdjson { namespace internal { /*! -implements the Grisu2 algorithm for binary to decimal floating-point -conversion. -Adapted from JSON for Modern C++ - -This implementation is a slightly modified version of the reference -implementation which may be obtained from -http://florian.loitsch.com/publications (bench.tar.gz). -The code is distributed under the MIT license, Copyright (c) 2009 Florian -Loitsch. For a detailed description of the algorithm see: [1] Loitsch, "Printing -Floating-Point Numbers Quickly and Accurately with Integers", Proceedings of the -ACM SIGPLAN 2010 Conference on Programming Language Design and Implementation, -PLDI 2010 [2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and -Accurately", Proceedings of the ACM SIGPLAN 1996 Conference on Programming -Language Design and Implementation, PLDI 1996 -*/ -namespace dtoa_impl { - -template -Target reinterpret_bits(const Source source) { - static_assert(sizeof(Target) == sizeof(Source), "size mismatch"); - - Target target; - std::memcpy(&target, &source, sizeof(Source)); - return target; -} - -struct diyfp // f * 2^e -{ - static constexpr int kPrecision = 64; // = q - - std::uint64_t f = 0; - int e = 0; +Implements the Dragonbox algorithm for binary to decimal floating-point +conversion (shortest round-trip representation of a double). - constexpr diyfp(std::uint64_t f_, int e_) noexcept : f(f_), e(e_) {} +The digit-generation core below is a self-contained port of Junekey Jeon's +reference "simple_dragonbox" implementation, specialized to IEEE-754 binary64 +and de-templated to match simdjson's style. Only the shortest-representation +path with the default (nearest, ties-to-even) rounding is kept. - /*! - @brief returns x - y - @pre x.e == y.e and x.f >= y.f - */ - static diyfp sub(const diyfp &x, const diyfp &y) noexcept { +Dragonbox: https://github.com/jk-jeon/dragonbox +Copyright 2020-2025 Junekey Jeon (and contributors). - return {x.f - y.f, x.e}; - } +The original is dual-licensed; this port is used under the terms of the +Boost Software License, Version 1.0 (https://www.boost.org/LICENSE_1_0.txt). - /*! - @brief returns x * y - @note The result is rounded. (Only the upper q bits are returned.) - */ - static diyfp mul(const diyfp &x, const diyfp &y) noexcept { - static_assert(kPrecision == 64, "internal error"); +For the algorithm itself see: +[1] Junekey Jeon, "Dragonbox: A New Floating-Point Binary-to-Decimal Conversion Algorithm" (2022). - // Computes: - // f = round((x.f * y.f) / 2^q) - // e = x.e + y.e + q - - // Emulate the 64-bit * 64-bit multiplication: - // - // p = u * v - // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi) - // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + - // 2^64 (u_hi v_hi ) = (p0 ) + 2^32 ((p1 ) + (p2 )) - // + 2^64 (p3 ) = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + - // 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 ) = - // (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + - // p2_hi + p3) = (p0_lo ) + 2^32 (Q ) + 2^64 (H ) = (p0_lo ) + - // 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H ) - // - // (Since Q might be larger than 2^32 - 1) - // - // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H) - // - // (Q_hi + H does not overflow a 64-bit int) - // - // = p_lo + 2^64 p_hi - - const std::uint64_t u_lo = x.f & 0xFFFFFFFFu; - const std::uint64_t u_hi = x.f >> 32u; - const std::uint64_t v_lo = y.f & 0xFFFFFFFFu; - const std::uint64_t v_hi = y.f >> 32u; - - const std::uint64_t p0 = u_lo * v_lo; - const std::uint64_t p1 = u_lo * v_hi; - const std::uint64_t p2 = u_hi * v_lo; - const std::uint64_t p3 = u_hi * v_hi; - - const std::uint64_t p0_hi = p0 >> 32u; - const std::uint64_t p1_lo = p1 & 0xFFFFFFFFu; - const std::uint64_t p1_hi = p1 >> 32u; - const std::uint64_t p2_lo = p2 & 0xFFFFFFFFu; - const std::uint64_t p2_hi = p2 >> 32u; - - std::uint64_t Q = p0_hi + p1_lo + p2_lo; - - // The full product might now be computed as - // - // p_hi = p3 + p2_hi + p1_hi + (Q >> 32) - // p_lo = p0_lo + (Q << 32) - // - // But in this particular case here, the full p_lo is not required. - // Effectively we only need to add the highest bit in p_lo to p_hi (and - // Q_hi + 1 does not overflow). - - Q += std::uint64_t{1} << (64u - 32u - 1u); // round, ties up - - const std::uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32u); - - return {h, x.e + y.e + 64}; - } - - /*! - @brief normalize x such that the significand is >= 2^(q-1) - @pre x.f != 0 - */ - static diyfp normalize(diyfp x) noexcept { - - while ((x.f >> 63u) == 0) { - x.f <<= 1u; - x.e--; - } - - return x; - } - - /*! - @brief normalize x such that the result has the exponent E - @pre e >= x.e and the upper e - x.e bits of x.f must be zero. - */ - static diyfp normalize_to(const diyfp &x, - const int target_exponent) noexcept { - const int delta = x.e - target_exponent; - - return {x.f << delta, target_exponent}; - } -}; - -struct boundaries { - diyfp w; - diyfp minus; - diyfp plus; -}; - -/*! -Compute the (normalized) diyfp representing the input number 'value' and its -boundaries. -@pre value must be finite and positive +The shortest decimal digits produced here are then laid out into the familiar +printf("%g")-style text by format_buffer(), which is unchanged from the previous +Grisu2-based implementation, so the emitted strings are identical except that +Dragonbox always yields the (sometimes shorter) shortest representation. */ -template boundaries compute_boundaries(FloatType value) { +namespace dtoa_impl { - // Convert the IEEE representation into a diyfp. - // - // If v is denormal: - // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) - // If v is normalized: - // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) +// 128-bit helpers (no compiler intrinsics, so the code stays portable). +struct uint128 { + std::uint64_t high; + std::uint64_t low; +}; - static_assert(std::numeric_limits::is_iec559, - "internal error: dtoa_short requires an IEEE-754 " - "floating-point implementation"); +inline std::uint64_t rotr64(std::uint64_t n, unsigned r) noexcept { + r &= 63; + return (n >> r) | (n << ((64 - r) & 63)); +} - constexpr int kPrecision = - std::numeric_limits::digits; // = p (includes the hidden bit) - constexpr int kBias = - std::numeric_limits::max_exponent - 1 + (kPrecision - 1); - constexpr int kMinExp = 1 - kBias; - constexpr std::uint64_t kHiddenBit = std::uint64_t{1} - << (kPrecision - 1); // = 2^(p-1) +inline std::uint64_t umul64(std::uint32_t x, std::uint32_t y) noexcept { + return x * std::uint64_t(y); +} - using bits_type = typename std::conditional::type; +// 64x64 -> 128 bit multiplication. +inline uint128 umul128(std::uint64_t x, std::uint64_t y) noexcept { +#if defined(__SIZEOF_INT128__) + const __uint128_t p = static_cast<__uint128_t>(x)*y; + return {std::uint64_t(p>>64), std::uint64_t(p)}; +#else // using fallback on 32-bit targets and MSVC + const std::uint32_t a = std::uint32_t(x >> 32); + const std::uint32_t b = std::uint32_t(x); + const std::uint32_t c = std::uint32_t(y >> 32); + const std::uint32_t d = std::uint32_t(y); - const std::uint64_t bits = reinterpret_bits(value); - const std::uint64_t E = bits >> (kPrecision - 1); - const std::uint64_t F = bits & (kHiddenBit - 1); + const std::uint64_t ac = umul64(a, c); + const std::uint64_t bc = umul64(b, c); + const std::uint64_t ad = umul64(a, d); + const std::uint64_t bd = umul64(b, d); - const bool is_denormal = E == 0; - const diyfp v = is_denormal - ? diyfp(F, kMinExp) - : diyfp(F + kHiddenBit, static_cast(E) - kBias); + const std::uint64_t intermediate = + (bd >> 32) + std::uint32_t(ad) + std::uint32_t(bc); - // Compute the boundaries m- and m+ of the floating-point value - // v = f * 2^e. - // - // Determine v- and v+, the floating-point predecessor and successor if v, - // respectively. - // - // v- = v - 2^e if f != 2^(p-1) or e == e_min (A) - // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B) - // - // v+ = v + 2^e - // - // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_ - // between m- and m+ round to v, regardless of how the input rounding - // algorithm breaks ties. - // - // ---+-------------+-------------+-------------+-------------+--- (A) - // v- m- v m+ v+ - // - // -----------------+------+------+-------------+-------------+--- (B) - // v- m- v m+ v+ + return {ac + (intermediate >> 32) + (ad >> 32) + (bc >> 32), + (intermediate << 32) + std::uint32_t(bd)}; +#endif +} - const bool lower_boundary_is_closer = F == 0 && E > 1; - const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); - const diyfp m_minus = lower_boundary_is_closer - ? diyfp(4 * v.f - 1, v.e - 2) // (B) - : diyfp(2 * v.f - 1, v.e - 1); // (A) +// High 64 bits of a 64x64 -> 128 bit multiplication. +inline std::uint64_t umul128_upper64(std::uint64_t x, std::uint64_t y) noexcept { +#if defined(__SIZEOF_INT128__) + return std::uint64_t((static_cast<__uint128_t>(x)*y)>>64); +#else // using fallback on 32-bit targets and MSVC + const std::uint32_t a = std::uint32_t(x >> 32); + const std::uint32_t b = std::uint32_t(x); + const std::uint32_t c = std::uint32_t(y >> 32); + const std::uint32_t d = std::uint32_t(y); - // Determine the normalized w+ = m+. - const diyfp w_plus = diyfp::normalize(m_plus); + const std::uint64_t ac = umul64(a, c); + const std::uint64_t bc = umul64(b, c); + const std::uint64_t ad = umul64(a, d); + const std::uint64_t bd = umul64(b, d); - // Determine w- = m- such that e_(w-) = e_(w+). - const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e); + const std::uint64_t intermediate = + (bd >> 32) + std::uint32_t(ad) + std::uint32_t(bc); - return {diyfp::normalize(v), w_minus, w_plus}; + return ac + (intermediate >> 32) + (ad >> 32) + (bc >> 32); +#endif } -// Given normalized diyfp w, Grisu needs to find a (normalized) cached -// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies -// within a certain range [alpha, gamma] (Definition 3.2 from [1]) -// -// alpha <= e = e_c + e_w + q <= gamma -// -// or -// -// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q -// <= f_c * f_w * 2^gamma -// -// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies -// -// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma -// -// or -// -// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma) -// -// The choice of (alpha,gamma) determines the size of the table and the form of -// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well -// in practice: -// -// The idea is to cut the number c * w = f * 2^e into two parts, which can be -// processed independently: An integral part p1, and a fractional part p2: -// -// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e -// = (f div 2^-e) + (f mod 2^-e) * 2^e -// = p1 + p2 * 2^e -// -// The conversion of p1 into decimal form requires a series of divisions and -// modulos by (a power of) 10. These operations are faster for 32-bit than for -// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be -// achieved by choosing -// -// -e >= 32 or e <= -32 := gamma -// -// In order to convert the fractional part -// -// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ... -// -// into decimal form, the fraction is repeatedly multiplied by 10 and the digits -// d[-i] are extracted in order: -// -// (10 * p2) div 2^-e = d[-1] -// (10 * p2) mod 2^-e = d[-2] / 10^1 + ... -// -// The multiplication by 10 must not overflow. It is sufficient to choose -// -// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64. -// -// Since p2 = f mod 2^-e < 2^-e, -// -// -e <= 60 or e >= -60 := alpha - -constexpr int kAlpha = -60; -constexpr int kGamma = -32; +// Upper 128 bits of a 64 x 128 -> 192 bit multiplication. +inline uint128 umul192_upper128(std::uint64_t x, uint128 y) noexcept { + uint128 r = umul128(x, y.high); + const std::uint64_t add = umul128_upper64(x, y.low); + const std::uint64_t sum = r.low + add; + r.high += (sum < r.low) ? 1 : 0; + r.low = sum; + return r; +} + +// Lower 128 bits of a 64 x 128 -> 192 bit multiplication. +inline uint128 umul192_lower128(std::uint64_t x, uint128 y) noexcept { + const std::uint64_t high = x * y.high; + const uint128 high_low = umul128(x, y.low); + return {high + high_low.high, high_low.low}; +} + +// Integer log approximations (exact over the range of inputs we feed them). +inline int floor_log10_pow2(int e) noexcept { return (e * 315653) >> 20; } +inline int floor_log2_pow10(int e) noexcept { return (e * 1741647) >> 19; } +inline int floor_log10_pow2_minus_log10_4_over_3(int e) noexcept { + return (e * 631305 - 261663) >> 21; +} + +// Format constants for IEEE-754 binary64, plus the precomputed cache of powers of ten. +static constexpr int kappa = 2; +static constexpr int significand_bits = 52; +static constexpr int total_bits = 64; +static constexpr int min_exponent = -1022; +static constexpr int exponent_bias = -1023; +static constexpr int cache_min_k = -292; +static constexpr int big_divisor = 1000; // 10^(kappa + 1) +static constexpr int small_divisor = 100; // 10^kappa +static constexpr int case_shorter_interval_left_endpoint_lower_threshold = 2; +static constexpr int case_shorter_interval_left_endpoint_upper_threshold = 3; +static constexpr int shorter_interval_tie_lower_threshold = -77; +static constexpr int shorter_interval_tie_upper_threshold = -77; + +// cache[i] holds a 128-bit approximation of a power of ten; indexed by +// (-minus_k - cache_min_k). Taken verbatim from the Dragonbox reference. +static constexpr uint128 cache[619] = { + {0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7b}, + {0x9faacf3df73609b1, 0x77b191618c54e9ad}, + {0xc795830d75038c1d, 0xd59df5b9ef6a2418}, + {0xf97ae3d0d2446f25, 0x4b0573286b44ad1e}, + {0x9becce62836ac577, 0x4ee367f9430aec33}, + {0xc2e801fb244576d5, 0x229c41f793cda740}, + {0xf3a20279ed56d48a, 0x6b43527578c11110}, + {0x9845418c345644d6, 0x830a13896b78aaaa}, + {0xbe5691ef416bd60c, 0x23cc986bc656d554}, + {0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa9}, + {0x94b3a202eb1c3f39, 0x7bf7d71432f3d6aa}, + {0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc54}, + {0xe858ad248f5c22c9, 0xd1b3400f8f9cff69}, + {0x91376c36d99995be, 0x23100809b9c21fa2}, + {0xb58547448ffffb2d, 0xabd40a0c2832a78b}, + {0xe2e69915b3fff9f9, 0x16c90c8f323f516d}, + {0x8dd01fad907ffc3b, 0xae3da7d97f6792e4}, + {0xb1442798f49ffb4a, 0x99cd11cfdf41779d}, + {0xdd95317f31c7fa1d, 0x40405643d711d584}, + {0x8a7d3eef7f1cfc52, 0x482835ea666b2573}, + {0xad1c8eab5ee43b66, 0xda3243650005eed0}, + {0xd863b256369d4a40, 0x90bed43e40076a83}, + {0x873e4f75e2224e68, 0x5a7744a6e804a292}, + {0xa90de3535aaae202, 0x711515d0a205cb37}, + {0xd3515c2831559a83, 0x0d5a5b44ca873e04}, + {0x8412d9991ed58091, 0xe858790afe9486c3}, + {0xa5178fff668ae0b6, 0x626e974dbe39a873}, + {0xce5d73ff402d98e3, 0xfb0a3d212dc81290}, + {0x80fa687f881c7f8e, 0x7ce66634bc9d0b9a}, + {0xa139029f6a239f72, 0x1c1fffc1ebc44e81}, + {0xc987434744ac874e, 0xa327ffb266b56221}, + {0xfbe9141915d7a922, 0x4bf1ff9f0062baa9}, + {0x9d71ac8fada6c9b5, 0x6f773fc3603db4aa}, + {0xc4ce17b399107c22, 0xcb550fb4384d21d4}, + {0xf6019da07f549b2b, 0x7e2a53a146606a49}, + {0x99c102844f94e0fb, 0x2eda7444cbfc426e}, + {0xc0314325637a1939, 0xfa911155fefb5309}, + {0xf03d93eebc589f88, 0x793555ab7eba27cb}, + {0x96267c7535b763b5, 0x4bc1558b2f3458df}, + {0xbbb01b9283253ca2, 0x9eb1aaedfb016f17}, + {0xea9c227723ee8bcb, 0x465e15a979c1cadd}, + {0x92a1958a7675175f, 0x0bfacd89ec191eca}, + {0xb749faed14125d36, 0xcef980ec671f667c}, + {0xe51c79a85916f484, 0x82b7e12780e7401b}, + {0x8f31cc0937ae58d2, 0xd1b2ecb8b0908811}, + {0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa16}, + {0xdfbdcece67006ac9, 0x67a791e093e1d49b}, + {0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e1}, + {0xaecc49914078536d, 0x58fae9f773886e19}, + {0xda7f5bf590966848, 0xaf39a475506a899f}, + {0x888f99797a5e012d, 0x6d8406c952429604}, + {0xaab37fd7d8f58178, 0xc8e5087ba6d33b84}, + {0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a65}, + {0x855c3be0a17fcd26, 0x5cf2eea09a550680}, + {0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481f}, + {0xd0601d8efc57b08b, 0xf13b94daf124da27}, + {0x823c12795db6ce57, 0x76c53d08d6b70859}, + {0xa2cb1717b52481ed, 0x54768c4b0c64ca6f}, + {0xcb7ddcdda26da268, 0xa9942f5dcf7dfd0a}, + {0xfe5d54150b090b02, 0xd3f93b35435d7c4d}, + {0x9efa548d26e5a6e1, 0xc47bc5014a1a6db0}, + {0xc6b8e9b0709f109a, 0x359ab6419ca1091c}, + {0xf867241c8cc6d4c0, 0xc30163d203c94b63}, + {0x9b407691d7fc44f8, 0x79e0de63425dcf1e}, + {0xc21094364dfb5636, 0x985915fc12f542e5}, + {0xf294b943e17a2bc4, 0x3e6f5b7b17b2939e}, + {0x979cf3ca6cec5b5a, 0xa705992ceecf9c43}, + {0xbd8430bd08277231, 0x50c6ff782a838354}, + {0xece53cec4a314ebd, 0xa4f8bf5635246429}, + {0x940f4613ae5ed136, 0x871b7795e136be9a}, + {0xb913179899f68584, 0x28e2557b59846e40}, + {0xe757dd7ec07426e5, 0x331aeada2fe589d0}, + {0x9096ea6f3848984f, 0x3ff0d2c85def7622}, + {0xb4bca50b065abe63, 0x0fed077a756b53aa}, + {0xe1ebce4dc7f16dfb, 0xd3e8495912c62895}, + {0x8d3360f09cf6e4bd, 0x64712dd7abbbd95d}, + {0xb080392cc4349dec, 0xbd8d794d96aacfb4}, + {0xdca04777f541c567, 0xecf0d7a0fc5583a1}, + {0x89e42caaf9491b60, 0xf41686c49db57245}, + {0xac5d37d5b79b6239, 0x311c2875c522ced6}, + {0xd77485cb25823ac7, 0x7d633293366b828c}, + {0x86a8d39ef77164bc, 0xae5dff9c02033198}, + {0xa8530886b54dbdeb, 0xd9f57f830283fdfd}, + {0xd267caa862a12d66, 0xd072df63c324fd7c}, + {0x8380dea93da4bc60, 0x4247cb9e59f71e6e}, + {0xa46116538d0deb78, 0x52d9be85f074e609}, + {0xcd795be870516656, 0x67902e276c921f8c}, + {0x806bd9714632dff6, 0x00ba1cd8a3db53b7}, + {0xa086cfcd97bf97f3, 0x80e8a40eccd228a5}, + {0xc8a883c0fdaf7df0, 0x6122cd128006b2ce}, + {0xfad2a4b13d1b5d6c, 0x796b805720085f82}, + {0x9cc3a6eec6311a63, 0xcbe3303674053bb1}, + {0xc3f490aa77bd60fc, 0xbedbfc4411068a9d}, + {0xf4f1b4d515acb93b, 0xee92fb5515482d45}, + {0x991711052d8bf3c5, 0x751bdd152d4d1c4b}, + {0xbf5cd54678eef0b6, 0xd262d45a78a0635e}, + {0xef340a98172aace4, 0x86fb897116c87c35}, + {0x9580869f0e7aac0e, 0xd45d35e6ae3d4da1}, + {0xbae0a846d2195712, 0x8974836059cca10a}, + {0xe998d258869facd7, 0x2bd1a438703fc94c}, + {0x91ff83775423cc06, 0x7b6306a34627ddd0}, + {0xb67f6455292cbf08, 0x1a3bc84c17b1d543}, + {0xe41f3d6a7377eeca, 0x20caba5f1d9e4a94}, + {0x8e938662882af53e, 0x547eb47b7282ee9d}, + {0xb23867fb2a35b28d, 0xe99e619a4f23aa44}, + {0xdec681f9f4c31f31, 0x6405fa00e2ec94d5}, + {0x8b3c113c38f9f37e, 0xde83bc408dd3dd05}, + {0xae0b158b4738705e, 0x9624ab50b148d446}, + {0xd98ddaee19068c76, 0x3badd624dd9b0958}, + {0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d7}, + {0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4d}, + {0xd47487cc8470652b, 0x7647c32000696720}, + {0x84c8d4dfd2c63f3b, 0x29ecd9f40041e074}, + {0xa5fb0a17c777cf09, 0xf468107100525891}, + {0xcf79cc9db955c2cc, 0x7182148d4066eeb5}, + {0x81ac1fe293d599bf, 0xc6f14cd848405531}, + {0xa21727db38cb002f, 0xb8ada00e5a506a7d}, + {0xca9cf1d206fdc03b, 0xa6d90811f0e4851d}, + {0xfd442e4688bd304a, 0x908f4a166d1da664}, + {0x9e4a9cec15763e2e, 0x9a598e4e043287ff}, + {0xc5dd44271ad3cdba, 0x40eff1e1853f29fe}, + {0xf7549530e188c128, 0xd12bee59e68ef47d}, + {0x9a94dd3e8cf578b9, 0x82bb74f8301958cf}, + {0xc13a148e3032d6e7, 0xe36a52363c1faf02}, + {0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac2}, + {0x96f5600f15a7b7e5, 0x29ab103a5ef8c0ba}, + {0xbcb2b812db11a5de, 0x7415d448f6b6f0e8}, + {0xebdf661791d60f56, 0x111b495b3464ad22}, + {0x936b9fcebb25c995, 0xcab10dd900beec35}, + {0xb84687c269ef3bfb, 0x3d5d514f40eea743}, + {0xe65829b3046b0afa, 0x0cb4a5a3112a5113}, + {0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ac}, + {0xb3f4e093db73a093, 0x59ed216765690f57}, + {0xe0f218b8d25088b8, 0x306869c13ec3532d}, + {0x8c974f7383725573, 0x1e414218c73a13fc}, + {0xafbd2350644eeacf, 0xe5d1929ef90898fb}, + {0xdbac6c247d62a583, 0xdf45f746b74abf3a}, + {0x894bc396ce5da772, 0x6b8bba8c328eb784}, + {0xab9eb47c81f5114f, 0x066ea92f3f326565}, + {0xd686619ba27255a2, 0xc80a537b0efefebe}, + {0x8613fd0145877585, 0xbd06742ce95f5f37}, + {0xa798fc4196e952e7, 0x2c48113823b73705}, + {0xd17f3b51fca3a7a0, 0xf75a15862ca504c6}, + {0x82ef85133de648c4, 0x9a984d73dbe722fc}, + {0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebbb}, + {0xcc963fee10b7d1b3, 0x318df905079926a9}, + {0xffbbcfe994e5c61f, 0xfdf17746497f7053}, + {0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa634}, + {0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc1}, + {0xf9bd690a1b68637b, 0x3dfdce7aa3c673b1}, + {0x9c1661a651213e2d, 0x06bea10ca65c084f}, + {0xc31bfa0fe5698db8, 0x486e494fcff30a63}, + {0xf3e2f893dec3f126, 0x5a89dba3c3efccfb}, + {0x986ddb5c6b3a76b7, 0xf89629465a75e01d}, + {0xbe89523386091465, 0xf6bbb397f1135824}, + {0xee2ba6c0678b597f, 0x746aa07ded582e2d}, + {0x94db483840b717ef, 0xa8c2a44eb4571cdd}, + {0xba121a4650e4ddeb, 0x92f34d62616ce414}, + {0xe896a0d7e51e1566, 0x77b020baf9c81d18}, + {0x915e2486ef32cd60, 0x0ace1474dc1d122f}, + {0xb5b5ada8aaff80b8, 0x0d819992132456bb}, + {0xe3231912d5bf60e6, 0x10e1fff697ed6c6a}, + {0x8df5efabc5979c8f, 0xca8d3ffa1ef463c2}, + {0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb3}, + {0xddd0467c64bce4a0, 0xac7cb3f6d05ddbdf}, + {0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96c}, + {0xad4ab7112eb3929d, 0x86c16c98d2c953c7}, + {0xd89d64d57a607744, 0xe871c7bf077ba8b8}, + {0x87625f056c7c4a8b, 0x11471cd764ad4973}, + {0xa93af6c6c79b5d2d, 0xd598e40d3dd89bd0}, + {0xd389b47879823479, 0x4aff1d108d4ec2c4}, + {0x843610cb4bf160cb, 0xcedf722a585139bb}, + {0xa54394fe1eedb8fe, 0xc2974eb4ee658829}, + {0xce947a3da6a9273e, 0x733d226229feea33}, + {0x811ccc668829b887, 0x0806357d5a3f5260}, + {0xa163ff802a3426a8, 0xca07c2dcb0cf26f8}, + {0xc9bcff6034c13052, 0xfc89b393dd02f0b6}, + {0xfc2c3f3841f17c67, 0xbbac2078d443ace3}, + {0x9d9ba7832936edc0, 0xd54b944b84aa4c0e}, + {0xc5029163f384a931, 0x0a9e795e65d4df12}, + {0xf64335bcf065d37d, 0x4d4617b5ff4a16d6}, + {0x99ea0196163fa42e, 0x504bced1bf8e4e46}, + {0xc06481fb9bcf8d39, 0xe45ec2862f71e1d7}, + {0xf07da27a82c37088, 0x5d767327bb4e5a4d}, + {0x964e858c91ba2655, 0x3a6a07f8d510f870}, + {0xbbe226efb628afea, 0x890489f70a55368c}, + {0xeadab0aba3b2dbe5, 0x2b45ac74ccea842f}, + {0x92c8ae6b464fc96f, 0x3b0b8bc90012929e}, + {0xb77ada0617e3bbcb, 0x09ce6ebb40173745}, + {0xe55990879ddcaabd, 0xcc420a6a101d0516}, + {0x8f57fa54c2a9eab6, 0x9fa946824a12232e}, + {0xb32df8e9f3546564, 0x47939822dc96abfa}, + {0xdff9772470297ebd, 0x59787e2b93bc56f8}, + {0x8bfbea76c619ef36, 0x57eb4edb3c55b65b}, + {0xaefae51477a06b03, 0xede622920b6b23f2}, + {0xdab99e59958885c4, 0xe95fab368e45ecee}, + {0x88b402f7fd75539b, 0x11dbcb0218ebb415}, + {0xaae103b5fcd2a881, 0xd652bdc29f26a11a}, + {0xd59944a37c0752a2, 0x4be76d3346f04960}, + {0x857fcae62d8493a5, 0x6f70a4400c562ddc}, + {0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb953}, + {0xd097ad07a71f26b2, 0x7e2000a41346a7a8}, + {0x825ecc24c873782f, 0x8ed400668c0c28c9}, + {0xa2f67f2dfa90563b, 0x728900802f0f32fb}, + {0xcbb41ef979346bca, 0x4f2b40a03ad2ffba}, + {0xfea126b7d78186bc, 0xe2f610c84987bfa9}, + {0x9f24b832e6b0f436, 0x0dd9ca7d2df4d7ca}, + {0xc6ede63fa05d3143, 0x91503d1c79720dbc}, + {0xf8a95fcf88747d94, 0x75a44c6397ce912b}, + {0x9b69dbe1b548ce7c, 0xc986afbe3ee11abb}, + {0xc24452da229b021b, 0xfbe85badce996169}, + {0xf2d56790ab41c2a2, 0xfae27299423fb9c4}, + {0x97c560ba6b0919a5, 0xdccd879fc967d41b}, + {0xbdb6b8e905cb600f, 0x5400e987bbc1c921}, + {0xed246723473e3813, 0x290123e9aab23b69}, + {0x9436c0760c86e30b, 0xf9a0b6720aaf6522}, + {0xb94470938fa89bce, 0xf808e40e8d5b3e6a}, + {0xe7958cb87392c2c2, 0xb60b1d1230b20e05}, + {0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c3}, + {0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af4}, + {0xe2280b6c20dd5232, 0x25c6da63c38de1b1}, + {0x8d590723948a535f, 0x579c487e5a38ad0f}, + {0xb0af48ec79ace837, 0x2d835a9df0c6d852}, + {0xdcdb1b2798182244, 0xf8e431456cf88e66}, + {0x8a08f0f8bf0f156b, 0x1b8e9ecb641b5900}, + {0xac8b2d36eed2dac5, 0xe272467e3d222f40}, + {0xd7adf884aa879177, 0x5b0ed81dcc6abb10}, + {0x86ccbb52ea94baea, 0x98e947129fc2b4ea}, + {0xa87fea27a539e9a5, 0x3f2398d747b36225}, + {0xd29fe4b18e88640e, 0x8eec7f0d19a03aae}, + {0x83a3eeeef9153e89, 0x1953cf68300424ad}, + {0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd8}, + {0xcdb02555653131b6, 0x3792f412cb06794e}, + {0x808e17555f3ebf11, 0xe2bbd88bbee40bd1}, + {0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec5}, + {0xc8de047564d20a8b, 0xf245825a5a445276}, + {0xfb158592be068d2e, 0xeed6e2f0f0d56713}, + {0x9ced737bb6c4183d, 0x55464dd69685606c}, + {0xc428d05aa4751e4c, 0xaa97e14c3c26b887}, + {0xf53304714d9265df, 0xd53dd99f4b3066a9}, + {0x993fe2c6d07b7fab, 0xe546a8038efe402a}, + {0xbf8fdb78849a5f96, 0xde98520472bdd034}, + {0xef73d256a5c0f77c, 0x963e66858f6d4441}, + {0x95a8637627989aad, 0xdde7001379a44aa9}, + {0xbb127c53b17ec159, 0x5560c018580d5d53}, + {0xe9d71b689dde71af, 0xaab8f01e6e10b4a7}, + {0x9226712162ab070d, 0xcab3961304ca70e9}, + {0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d23}, + {0xe45c10c42a2b3b05, 0x8cb89a7db77c506b}, + {0x8eb98a7a9a5b04e3, 0x77f3608e92adb243}, + {0xb267ed1940f1c61c, 0x55f038b237591ed4}, + {0xdf01e85f912e37a3, 0x6b6c46dec52f6689}, + {0x8b61313bbabce2c6, 0x2323ac4b3b3da016}, + {0xae397d8aa96c1b77, 0xabec975e0a0d081b}, + {0xd9c7dced53c72255, 0x96e7bd358c904a22}, + {0x881cea14545c7575, 0x7e50d64177da2e55}, + {0xaa242499697392d2, 0xdde50bd1d5d0b9ea}, + {0xd4ad2dbfc3d07787, 0x955e4ec64b44e865}, + {0x84ec3c97da624ab4, 0xbd5af13bef0b113f}, + {0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58f}, + {0xcfb11ead453994ba, 0x67de18eda5814af3}, + {0x81ceb32c4b43fcf4, 0x80eacf948770ced8}, + {0xa2425ff75e14fc31, 0xa1258379a94d028e}, + {0xcad2f7f5359a3b3e, 0x096ee45813a04331}, + {0xfd87b5f28300ca0d, 0x8bca9d6e188853fd}, + {0x9e74d1b791e07e48, 0x775ea264cf55347e}, + {0xc612062576589dda, 0x95364afe032a819e}, + {0xf79687aed3eec551, 0x3a83ddbd83f52205}, + {0x9abe14cd44753b52, 0xc4926a9672793543}, + {0xc16d9a0095928a27, 0x75b7053c0f178294}, + {0xf1c90080baf72cb1, 0x5324c68b12dd6339}, + {0x971da05074da7bee, 0xd3f6fc16ebca5e04}, + {0xbce5086492111aea, 0x88f4bb1ca6bcf585}, + {0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6}, + {0x9392ee8e921d5d07, 0x3aff322e62439fd0}, + {0xb877aa3236a4b449, 0x09befeb9fad487c3}, + {0xe69594bec44de15b, 0x4c2ebe687989a9b4}, + {0x901d7cf73ab0acd9, 0x0f9d37014bf60a11}, + {0xb424dc35095cd80f, 0x538484c19ef38c95}, + {0xe12e13424bb40e13, 0x2865a5f206b06fba}, + {0x8cbccc096f5088cb, 0xf93f87b7442e45d4}, + {0xafebff0bcb24aafe, 0xf78f69a51539d749}, + {0xdbe6fecebdedd5be, 0xb573440e5a884d1c}, + {0x89705f4136b4a597, 0x31680a88f8953031}, + {0xabcc77118461cefc, 0xfdc20d2b36ba7c3e}, + {0xd6bf94d5e57a42bc, 0x3d32907604691b4d}, + {0x8637bd05af6c69b5, 0xa63f9a49c2c1b110}, + {0xa7c5ac471b478423, 0x0fcf80dc33721d54}, + {0xd1b71758e219652b, 0xd3c36113404ea4a9}, + {0x83126e978d4fdf3b, 0x645a1cac083126ea}, + {0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4}, + {0xcccccccccccccccc, 0xcccccccccccccccd}, + {0x8000000000000000, 0x0000000000000000}, + {0xa000000000000000, 0x0000000000000000}, + {0xc800000000000000, 0x0000000000000000}, + {0xfa00000000000000, 0x0000000000000000}, + {0x9c40000000000000, 0x0000000000000000}, + {0xc350000000000000, 0x0000000000000000}, + {0xf424000000000000, 0x0000000000000000}, + {0x9896800000000000, 0x0000000000000000}, + {0xbebc200000000000, 0x0000000000000000}, + {0xee6b280000000000, 0x0000000000000000}, + {0x9502f90000000000, 0x0000000000000000}, + {0xba43b74000000000, 0x0000000000000000}, + {0xe8d4a51000000000, 0x0000000000000000}, + {0x9184e72a00000000, 0x0000000000000000}, + {0xb5e620f480000000, 0x0000000000000000}, + {0xe35fa931a0000000, 0x0000000000000000}, + {0x8e1bc9bf04000000, 0x0000000000000000}, + {0xb1a2bc2ec5000000, 0x0000000000000000}, + {0xde0b6b3a76400000, 0x0000000000000000}, + {0x8ac7230489e80000, 0x0000000000000000}, + {0xad78ebc5ac620000, 0x0000000000000000}, + {0xd8d726b7177a8000, 0x0000000000000000}, + {0x878678326eac9000, 0x0000000000000000}, + {0xa968163f0a57b400, 0x0000000000000000}, + {0xd3c21bcecceda100, 0x0000000000000000}, + {0x84595161401484a0, 0x0000000000000000}, + {0xa56fa5b99019a5c8, 0x0000000000000000}, + {0xcecb8f27f4200f3a, 0x0000000000000000}, + {0x813f3978f8940984, 0x4000000000000000}, + {0xa18f07d736b90be5, 0x5000000000000000}, + {0xc9f2c9cd04674ede, 0xa400000000000000}, + {0xfc6f7c4045812296, 0x4d00000000000000}, + {0x9dc5ada82b70b59d, 0xf020000000000000}, + {0xc5371912364ce305, 0x6c28000000000000}, + {0xf684df56c3e01bc6, 0xc732000000000000}, + {0x9a130b963a6c115c, 0x3c7f400000000000}, + {0xc097ce7bc90715b3, 0x4b9f100000000000}, + {0xf0bdc21abb48db20, 0x1e86d40000000000}, + {0x96769950b50d88f4, 0x1314448000000000}, + {0xbc143fa4e250eb31, 0x17d955a000000000}, + {0xeb194f8e1ae525fd, 0x5dcfab0800000000}, + {0x92efd1b8d0cf37be, 0x5aa1cae500000000}, + {0xb7abc627050305ad, 0xf14a3d9e40000000}, + {0xe596b7b0c643c719, 0x6d9ccd05d0000000}, + {0x8f7e32ce7bea5c6f, 0xe4820023a2000000}, + {0xb35dbf821ae4f38b, 0xdda2802c8a800000}, + {0xe0352f62a19e306e, 0xd50b2037ad200000}, + {0x8c213d9da502de45, 0x4526f422cc340000}, + {0xaf298d050e4395d6, 0x9670b12b7f410000}, + {0xdaf3f04651d47b4c, 0x3c0cdd765f114000}, + {0x88d8762bf324cd0f, 0xa5880a69fb6ac800}, + {0xab0e93b6efee0053, 0x8eea0d047a457a00}, + {0xd5d238a4abe98068, 0x72a4904598d6d880}, + {0x85a36366eb71f041, 0x47a6da2b7f864750}, + {0xa70c3c40a64e6c51, 0x999090b65f67d924}, + {0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d}, + {0x82818f1281ed449f, 0xbff8f10e7a8921a5}, + {0xa321f2d7226895c7, 0xaff72d52192b6a0e}, + {0xcbea6f8ceb02bb39, 0x9bf4f8a69f764491}, + {0xfee50b7025c36a08, 0x02f236d04753d5b5}, + {0x9f4f2726179a2245, 0x01d762422c946591}, + {0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef6}, + {0xf8ebad2b84e0d58b, 0xd2e0898765a7deb3}, + {0x9b934c3b330c8577, 0x63cc55f49f88eb30}, + {0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fc}, + {0xf316271c7fc3908a, 0x8bef464e3945ef7b}, + {0x97edd871cfda3a56, 0x97758bf0e3cbb5ad}, + {0xbde94e8e43d0c8ec, 0x3d52eeed1cbea318}, + {0xed63a231d4c4fb27, 0x4ca7aaa863ee4bde}, + {0x945e455f24fb1cf8, 0x8fe8caa93e74ef6b}, + {0xb975d6b6ee39e436, 0xb3e2fd538e122b45}, + {0xe7d34c64a9c85d44, 0x60dbbca87196b617}, + {0x90e40fbeea1d3a4a, 0xbc8955e946fe31ce}, + {0xb51d13aea4a488dd, 0x6babab6398bdbe42}, + {0xe264589a4dcdab14, 0xc696963c7eed2dd2}, + {0x8d7eb76070a08aec, 0xfc1e1de5cf543ca3}, + {0xb0de65388cc8ada8, 0x3b25a55f43294bcc}, + {0xdd15fe86affad912, 0x49ef0eb713f39ebf}, + {0x8a2dbf142dfcc7ab, 0x6e3569326c784338}, + {0xacb92ed9397bf996, 0x49c2c37f07965405}, + {0xd7e77a8f87daf7fb, 0xdc33745ec97be907}, + {0x86f0ac99b4e8dafd, 0x69a028bb3ded71a4}, + {0xa8acd7c0222311bc, 0xc40832ea0d68ce0d}, + {0xd2d80db02aabd62b, 0xf50a3fa490c30191}, + {0x83c7088e1aab65db, 0x792667c6da79e0fb}, + {0xa4b8cab1a1563f52, 0x577001b891185939}, + {0xcde6fd5e09abcf26, 0xed4c0226b55e6f87}, + {0x80b05e5ac60b6178, 0x544f8158315b05b5}, + {0xa0dc75f1778e39d6, 0x696361ae3db1c722}, + {0xc913936dd571c84c, 0x03bc3a19cd1e38ea}, + {0xfb5878494ace3a5f, 0x04ab48a04065c724}, + {0x9d174b2dcec0e47b, 0x62eb0d64283f9c77}, + {0xc45d1df942711d9a, 0x3ba5d0bd324f8395}, + {0xf5746577930d6500, 0xca8f44ec7ee3647a}, + {0x9968bf6abbe85f20, 0x7e998b13cf4e1ecc}, + {0xbfc2ef456ae276e8, 0x9e3fedd8c321a67f}, + {0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101f}, + {0x95d04aee3b80ece5, 0xbba1f1d158724a13}, + {0xbb445da9ca61281f, 0x2a8a6e45ae8edc98}, + {0xea1575143cf97226, 0xf52d09d71a3293be}, + {0x924d692ca61be758, 0x593c2626705f9c57}, + {0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836d}, + {0xe498f455c38b997a, 0x0b6dfb9c0f956448}, + {0x8edf98b59a373fec, 0x4724bd4189bd5ead}, + {0xb2977ee300c50fe7, 0x58edec91ec2cb658}, + {0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ee}, + {0x8b865b215899f46c, 0xbd79e0d20082ee75}, + {0xae67f1e9aec07187, 0xecd8590680a3aa12}, + {0xda01ee641a708de9, 0xe80e6f4820cc9496}, + {0x884134fe908658b2, 0x3109058d147fdcde}, + {0xaa51823e34a7eede, 0xbd4b46f0599fd416}, + {0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91b}, + {0x850fadc09923329e, 0x03e2cf6bc604ddb1}, + {0xa6539930bf6bff45, 0x84db8346b786151d}, + {0xcfe87f7cef46ff16, 0xe612641865679a64}, + {0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07f}, + {0xa26da3999aef7749, 0xe3be5e330f38f09e}, + {0xcb090c8001ab551c, 0x5cadf5bfd3072cc6}, + {0xfdcb4fa002162a63, 0x73d9732fc7c8f7f7}, + {0x9e9f11c4014dda7e, 0x2867e7fddcdd9afb}, + {0xc646d63501a1511d, 0xb281e1fd541501b9}, + {0xf7d88bc24209a565, 0x1f225a7ca91a4227}, + {0x9ae757596946075f, 0x3375788de9b06959}, + {0xc1a12d2fc3978937, 0x0052d6b1641c83af}, + {0xf209787bb47d6b84, 0xc0678c5dbd23a49b}, + {0x9745eb4d50ce6332, 0xf840b7ba963646e1}, + {0xbd176620a501fbff, 0xb650e5a93bc3d899}, + {0xec5d3fa8ce427aff, 0xa3e51f138ab4cebf}, + {0x93ba47c980e98cdf, 0xc66f336c36b10138}, + {0xb8a8d9bbe123f017, 0xb80b0047445d4185}, + {0xe6d3102ad96cec1d, 0xa60dc059157491e6}, + {0x9043ea1ac7e41392, 0x87c89837ad68db30}, + {0xb454e4a179dd1877, 0x29babe4598c311fc}, + {0xe16a1dc9d8545e94, 0xf4296dd6fef3d67b}, + {0x8ce2529e2734bb1d, 0x1899e4a65f58660d}, + {0xb01ae745b101e9e4, 0x5ec05dcff72e7f90}, + {0xdc21a1171d42645d, 0x76707543f4fa1f74}, + {0x899504ae72497eba, 0x6a06494a791c53a9}, + {0xabfa45da0edbde69, 0x0487db9d17636893}, + {0xd6f8d7509292d603, 0x45a9d2845d3c42b7}, + {0x865b86925b9bc5c2, 0x0b8a2392ba45a9b3}, + {0xa7f26836f282b732, 0x8e6cac7768d7141f}, + {0xd1ef0244af2364ff, 0x3207d795430cd927}, + {0x8335616aed761f1f, 0x7f44e6bd49e807b9}, + {0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a7}, + {0xcd036837130890a1, 0x36dba887c37a8c10}, + {0x802221226be55a64, 0xc2494954da2c978a}, + {0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6d}, + {0xc83553c5c8965d3d, 0x6f92829494e5acc8}, + {0xfa42a8b73abbf48c, 0xcb772339ba1f17fa}, + {0x9c69a97284b578d7, 0xff2a760414536efc}, + {0xc38413cf25e2d70d, 0xfef5138519684abb}, + {0xf46518c2ef5b8cd1, 0x7eb258665fc25d6a}, + {0x98bf2f79d5993802, 0xef2f773ffbd97a62}, + {0xbeeefb584aff8603, 0xaafb550ffacfd8fb}, + {0xeeaaba2e5dbf6784, 0x95ba2a53f983cf39}, + {0x952ab45cfa97a0b2, 0xdd945a747bf26184}, + {0xba756174393d88df, 0x94f971119aeef9e5}, + {0xe912b9d1478ceb17, 0x7a37cd5601aab85e}, + {0x91abb422ccb812ee, 0xac62e055c10ab33b}, + {0xb616a12b7fe617aa, 0x577b986b314d600a}, + {0xe39c49765fdf9d94, 0xed5a7e85fda0b80c}, + {0x8e41ade9fbebc27d, 0x14588f13be847308}, + {0xb1d219647ae6b31c, 0x596eb2d8ae258fc9}, + {0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bc}, + {0x8aec23d680043bee, 0x25de7bb9480d5855}, + {0xada72ccc20054ae9, 0xaf561aa79a10ae6b}, + {0xd910f7ff28069da4, 0x1b2ba1518094da05}, + {0x87aa9aff79042286, 0x90fb44d2f05d0843}, + {0xa99541bf57452b28, 0x353a1607ac744a54}, + {0xd3fa922f2d1675f2, 0x42889b8997915ce9}, + {0x847c9b5d7c2e09b7, 0x69956135febada12}, + {0xa59bc234db398c25, 0x43fab9837e699096}, + {0xcf02b2c21207ef2e, 0x94f967e45e03f4bc}, + {0x8161afb94b44f57d, 0x1d1be0eebac278f6}, + {0xa1ba1ba79e1632dc, 0x6462d92a69731733}, + {0xca28a291859bbf93, 0x7d7b8f7503cfdcff}, + {0xfcb2cb35e702af78, 0x5cda735244c3d43f}, + {0x9defbf01b061adab, 0x3a0888136afa64a8}, + {0xc56baec21c7a1916, 0x088aaa1845b8fdd1}, + {0xf6c69a72a3989f5b, 0x8aad549e57273d46}, + {0x9a3c2087a63f6399, 0x36ac54e2f678864c}, + {0xc0cb28a98fcf3c7f, 0x84576a1bb416a7de}, + {0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d6}, + {0x969eb7c47859e743, 0x9f644ae5a4b1b326}, + {0xbc4665b596706114, 0x873d5d9f0dde1fef}, + {0xeb57ff22fc0c7959, 0xa90cb506d155a7eb}, + {0x9316ff75dd87cbd8, 0x09a7f12442d588f3}, + {0xb7dcbf5354e9bece, 0x0c11ed6d538aeb30}, + {0xe5d3ef282a242e81, 0x8f1668c8a86da5fb}, + {0x8fa475791a569d10, 0xf96e017d694487bd}, + {0xb38d92d760ec4455, 0x37c981dcc395a9ad}, + {0xe070f78d3927556a, 0x85bbe253f47b1418}, + {0x8c469ab843b89562, 0x93956d7478ccec8f}, + {0xaf58416654a6babb, 0x387ac8d1970027b3}, + {0xdb2e51bfe9d0696a, 0x06997b05fcc0319f}, + {0x88fcf317f22241e2, 0x441fece3bdf81f04}, + {0xab3c2fddeeaad25a, 0xd527e81cad7626c4}, + {0xd60b3bd56a5586f1, 0x8a71e223d8d3b075}, + {0x85c7056562757456, 0xf6872d5667844e4a}, + {0xa738c6bebb12d16c, 0xb428f8ac016561dc}, + {0xd106f86e69d785c7, 0xe13336d701beba53}, + {0x82a45b450226b39c, 0xecc0024661173474}, + {0xa34d721642b06084, 0x27f002d7f95d0191}, + {0xcc20ce9bd35c78a5, 0x31ec038df7b441f5}, + {0xff290242c83396ce, 0x7e67047175a15272}, + {0x9f79a169bd203e41, 0x0f0062c6e984d387}, + {0xc75809c42c684dd1, 0x52c07b78a3e60869}, + {0xf92e0c3537826145, 0xa7709a56ccdf8a83}, + {0x9bbcc7a142b17ccb, 0x88a66076400bb692}, + {0xc2abf989935ddbfe, 0x6acff893d00ea436}, + {0xf356f7ebf83552fe, 0x0583f6b8c4124d44}, + {0x98165af37b2153de, 0xc3727a337a8b704b}, + {0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5d}, + {0xeda2ee1c7064130c, 0x1162def06f79df74}, + {0x9485d4d1c63e8be7, 0x8addcb5645ac2ba9}, + {0xb9a74a0637ce2ee1, 0x6d953e2bd7173693}, + {0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0438}, + {0x910ab1d4db9914a0, 0x1d9c9892400a22a3}, + {0xb54d5e4a127f59c8, 0x2503beb6d00cab4c}, + {0xe2a0b5dc971f303a, 0x2e44ae64840fd61e}, + {0x8da471a9de737e24, 0x5ceaecfed289e5d3}, + {0xb10d8e1456105dad, 0x7425a83e872c5f48}, + {0xdd50f1996b947518, 0xd12f124e28f7771a}, + {0x8a5296ffe33cc92f, 0x82bd6b70d99aaa70}, + {0xace73cbfdc0bfb7b, 0x636cc64d1001550c}, + {0xd8210befd30efa5a, 0x3c47f7e05401aa4f}, + {0x8714a775e3e95c78, 0x65acfaec34810a72}, + {0xa8d9d1535ce3b396, 0x7f1839a741a14d0e}, + {0xd31045a8341ca07c, 0x1ede48111209a051}, + {0x83ea2b892091e44d, 0x934aed0aab460433}, + {0xa4e4b66b68b65d60, 0xf81da84d56178540}, + {0xce1de40642e3f4b9, 0x36251260ab9d668f}, + {0x80d2ae83e9ce78f3, 0xc1d72b7c6b42601a}, + {0xa1075a24e4421730, 0xb24cf65b8612f820}, + {0xc94930ae1d529cfc, 0xdee033f26797b628}, + {0xfb9b7cd9a4a7443c, 0x169840ef017da3b2}, + {0x9d412e0806e88aa5, 0x8e1f289560ee864f}, + {0xc491798a08a2ad4e, 0xf1a6f2bab92a27e3}, + {0xf5b5d7ec8acb58a2, 0xae10af696774b1dc}, + {0x9991a6f3d6bf1765, 0xacca6da1e0a8ef2a}, + {0xbff610b0cc6edd3f, 0x17fd090a58d32af4}, + {0xeff394dcff8a948e, 0xddfc4b4cef07f5b1}, + {0x95f83d0a1fb69cd9, 0x4abdaf101564f98f}, + {0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f2}, + {0xea53df5fd18d5513, 0x84c86189216dc5ee}, + {0x92746b9be2f8552c, 0x32fd3cf5b4e49bb5}, + {0xb7118682dbb66a77, 0x3fbc8c33221dc2a2}, + {0xe4d5e82392a40515, 0x0fabaf3feaa5334b}, + {0x8f05b1163ba6832d, 0x29cb4d87f2a7400f}, + {0xb2c71d5bca9023f8, 0x743e20e9ef511013}, + {0xdf78e4b2bd342cf6, 0x914da9246b255417}, + {0x8bab8eefb6409c1a, 0x1ad089b6c2f7548f}, + {0xae9672aba3d0c320, 0xa184ac2473b529b2}, + {0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741f}, + {0x8865899617fb1871, 0x7e2fa67c7a658893}, + {0xaa7eebfb9df9de8d, 0xddbb901b98feeab8}, + {0xd51ea6fa85785631, 0x552a74227f3ea566}, + {0x8533285c936b35de, 0xd53a88958f872760}, + {0xa67ff273b8460356, 0x8a892abaf368f138}, + {0xd01fef10a657842c, 0x2d2b7569b0432d86}, + {0x8213f56a67f6b29b, 0x9c3b29620e29fc74}, + {0xa298f2c501f45f42, 0x8349f3ba91b47b90}, + {0xcb3f2f7642717713, 0x241c70a936219a74}, + {0xfe0efb53d30dd4d7, 0xed238cd383aa0111}, + {0x9ec95d1463e8a506, 0xf4363804324a40ab}, + {0xc67bb4597ce2ce48, 0xb143c6053edcd0d6}, + {0xf81aa16fdc1b81da, 0xdd94b7868e94050b}, + {0x9b10a4e5e9913128, 0xca7cf2b4191c8327}, + {0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f1}, + {0xf24a01a73cf2dccf, 0xbc633b39673c8ced}, + {0x976e41088617ca01, 0xd5be0503e085d814}, + {0xbd49d14aa79dbc82, 0x4b2d8644d8a74e19}, + {0xec9c459d51852ba2, 0xddf8e7d60ed1219f}, + {0x93e1ab8252f33b45, 0xcabb90e5c942b504}, + {0xb8da1662e7b00a17, 0x3d6a751f3b936244}, + {0xe7109bfba19c0c9d, 0x0cc512670a783ad5}, + {0x906a617d450187e2, 0x27fb2b80668b24c6}, + {0xb484f9dc9641e9da, 0xb1f9f660802dedf7}, + {0xe1a63853bbd26451, 0x5e7873f8a0396974}, + {0x8d07e33455637eb2, 0xdb0b487b6423e1e9}, + {0xb049dc016abc5e5f, 0x91ce1a9a3d2cda63}, + {0xdc5c5301c56b75f7, 0x7641a140cc7810fc}, + {0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9e}, + {0xac2820d9623bf429, 0x546345fa9fbdcd45}, + {0xd732290fbacaf133, 0xa97c177947ad4096}, + {0x867f59a9d4bed6c0, 0x49ed8eabcccc485e}, + {0xa81f301449ee8c70, 0x5c68f256bfff5a75}, + {0xd226fc195c6a2f8c, 0x73832eec6fff3112}, + {0x83585d8fd9c25db7, 0xc831fd53c5ff7eac}, + {0xa42e74f3d032f525, 0xba3e7ca8b77f5e56}, + {0xcd3a1230c43fb26f, 0x28ce1bd2e55f35ec}, + {0x80444b5e7aa7cf85, 0x7980d163cf5b81b4}, + {0xa0555e361951c366, 0xd7e105bcc3326220}, + {0xc86ab5c39fa63440, 0x8dd9472bf3fefaa8}, + {0xfa856334878fc150, 0xb14f98f6f0feb952}, + {0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d4}, + {0xc3b8358109e84f07, 0x0a862f80ec4700c9}, + {0xf4a642e14c6262c8, 0xcd27bb612758c0fb}, + {0x98e7e9cccfbd7dbd, 0x8038d51cb897789d}, + {0xbf21e44003acdd2c, 0xe0470a63e6bd56c4}, + {0xeeea5d5004981478, 0x1858ccfce06cac75}, + {0x95527a5202df0ccb, 0x0f37801e0c43ebc9}, + {0xbaa718e68396cffd, 0xd30560258f54e6bb}, + {0xe950df20247c83fd, 0x47c6b82ef32a206a}, + {0x91d28b7416cdd27e, 0x4cdc331d57fa5442}, + {0xb6472e511c81471d, 0xe0133fe4adf8e953}, + {0xe3d8f9e563a198e5, 0x58180fddd97723a7}, + {0x8e679c2f5e44ff8f, 0x570f09eaa7ea7649}, + {0xb201833b35d63f73, 0x2cd2cc6551e513db}, + {0xde81e40a034bcf4f, 0xf8077f7ea65e58d2}, + {0x8b112e86420f6191, 0xfb04afaf27faf783}, + {0xadd57a27d29339f6, 0x79c5db9af1f9b564}, + {0xd94ad8b1c7380874, 0x18375281ae7822bd}, + {0x87cec76f1c830548, 0x8f2293910d0b15b6}, + {0xa9c2794ae3a3c69a, 0xb2eb3875504ddb23}, + {0xd433179d9c8cb841, 0x5fa60692a46151ec}, + {0x849feec281d7f328, 0xdbc7c41ba6bcd334}, + {0xa5c7ea73224deff3, 0x12b9b522906c0801}, + {0xcf39e50feae16bef, 0xd768226b34870a01}, + {0x81842f29f2cce375, 0xe6a1158300d46641}, + {0xa1e53af46f801c53, 0x60495ae3c1097fd1}, + {0xca5e89b18b602368, 0x385bb19cb14bdfc5}, + {0xfcf62c1dee382c42, 0x46729e03dd9ed7b6}, + {0x9e19db92b4e31ba9, 0x6c07a2c26a8346d2}, + {0xc5a05277621be293, 0xc7098b7305241886}, + {0xf70867153aa2db38, 0xb8cbee4fc66d1ea8} +}; -struct cached_power // c = f * 2^e ~= 10^k -{ - std::uint64_t f; - int e; - int k; +// Per-format helper routines (binary64 specializations of the Dragonbox steps). +struct compute_mul_result { + std::uint64_t integer_part; + bool is_integer; +}; +struct compute_mul_parity_result { + bool parity; + bool is_integer; }; -/*! -For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached -power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c -satisfies (Definition 3.2 from [1]) - alpha <= e_c + e + q <= gamma. -*/ -inline cached_power get_cached_power_for_binary_exponent(int e) { - // Now - // - // alpha <= e_c + e + q <= gamma (1) - // ==> f_c * 2^alpha <= c * 2^e * 2^q - // - // and since the c's are normalized, 2^(q-1) <= f_c, - // - // ==> 2^(q - 1 + alpha) <= c * 2^(e + q) - // ==> 2^(alpha - e - 1) <= c - // - // If c were an exact power of ten, i.e. c = 10^k, one may determine k as - // - // k = ceil( log_10( 2^(alpha - e - 1) ) ) - // = ceil( (alpha - e - 1) * log_10(2) ) - // - // From the paper: - // "In theory the result of the procedure could be wrong since c is rounded, - // and the computation itself is approximated [...]. In practice, however, - // this simple function is sufficient." - // - // For IEEE double precision floating-point numbers converted into - // normalized diyfp's w = f * 2^e, with q = 64, - // - // e >= -1022 (min IEEE exponent) - // -52 (p - 1) - // -52 (p - 1, possibly normalize denormal IEEE numbers) - // -11 (normalize the diyfp) - // = -1137 - // - // and - // - // e <= +1023 (max IEEE exponent) - // -52 (p - 1) - // -11 (normalize the diyfp) - // = 960 - // - // This binary exponent range [-1137,960] results in a decimal exponent - // range [-307,324]. One does not need to store a cached power for each - // k in this range. For each such k it suffices to find a cached power - // such that the exponent of the product lies in [alpha,gamma]. - // This implies that the difference of the decimal exponents of adjacent - // table entries must be less than or equal to - // - // floor( (gamma - alpha) * log_10(2) ) = 8. - // - // (A smaller distance gamma-alpha would require a larger table.) - - // NB: - // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. - - constexpr int kCachedPowersMinDecExp = -300; - constexpr int kCachedPowersDecStep = 8; - - static constexpr std::array kCachedPowers = {{ - {0xAB70FE17C79AC6CA, -1060, -300}, {0xFF77B1FCBEBCDC4F, -1034, -292}, - {0xBE5691EF416BD60C, -1007, -284}, {0x8DD01FAD907FFC3C, -980, -276}, - {0xD3515C2831559A83, -954, -268}, {0x9D71AC8FADA6C9B5, -927, -260}, - {0xEA9C227723EE8BCB, -901, -252}, {0xAECC49914078536D, -874, -244}, - {0x823C12795DB6CE57, -847, -236}, {0xC21094364DFB5637, -821, -228}, - {0x9096EA6F3848984F, -794, -220}, {0xD77485CB25823AC7, -768, -212}, - {0xA086CFCD97BF97F4, -741, -204}, {0xEF340A98172AACE5, -715, -196}, - {0xB23867FB2A35B28E, -688, -188}, {0x84C8D4DFD2C63F3B, -661, -180}, - {0xC5DD44271AD3CDBA, -635, -172}, {0x936B9FCEBB25C996, -608, -164}, - {0xDBAC6C247D62A584, -582, -156}, {0xA3AB66580D5FDAF6, -555, -148}, - {0xF3E2F893DEC3F126, -529, -140}, {0xB5B5ADA8AAFF80B8, -502, -132}, - {0x87625F056C7C4A8B, -475, -124}, {0xC9BCFF6034C13053, -449, -116}, - {0x964E858C91BA2655, -422, -108}, {0xDFF9772470297EBD, -396, -100}, - {0xA6DFBD9FB8E5B88F, -369, -92}, {0xF8A95FCF88747D94, -343, -84}, - {0xB94470938FA89BCF, -316, -76}, {0x8A08F0F8BF0F156B, -289, -68}, - {0xCDB02555653131B6, -263, -60}, {0x993FE2C6D07B7FAC, -236, -52}, - {0xE45C10C42A2B3B06, -210, -44}, {0xAA242499697392D3, -183, -36}, - {0xFD87B5F28300CA0E, -157, -28}, {0xBCE5086492111AEB, -130, -20}, - {0x8CBCCC096F5088CC, -103, -12}, {0xD1B71758E219652C, -77, -4}, - {0x9C40000000000000, -50, 4}, {0xE8D4A51000000000, -24, 12}, - {0xAD78EBC5AC620000, 3, 20}, {0x813F3978F8940984, 30, 28}, - {0xC097CE7BC90715B3, 56, 36}, {0x8F7E32CE7BEA5C70, 83, 44}, - {0xD5D238A4ABE98068, 109, 52}, {0x9F4F2726179A2245, 136, 60}, - {0xED63A231D4C4FB27, 162, 68}, {0xB0DE65388CC8ADA8, 189, 76}, - {0x83C7088E1AAB65DB, 216, 84}, {0xC45D1DF942711D9A, 242, 92}, - {0x924D692CA61BE758, 269, 100}, {0xDA01EE641A708DEA, 295, 108}, - {0xA26DA3999AEF774A, 322, 116}, {0xF209787BB47D6B85, 348, 124}, - {0xB454E4A179DD1877, 375, 132}, {0x865B86925B9BC5C2, 402, 140}, - {0xC83553C5C8965D3D, 428, 148}, {0x952AB45CFA97A0B3, 455, 156}, - {0xDE469FBD99A05FE3, 481, 164}, {0xA59BC234DB398C25, 508, 172}, - {0xF6C69A72A3989F5C, 534, 180}, {0xB7DCBF5354E9BECE, 561, 188}, - {0x88FCF317F22241E2, 588, 196}, {0xCC20CE9BD35C78A5, 614, 204}, - {0x98165AF37B2153DF, 641, 212}, {0xE2A0B5DC971F303A, 667, 220}, - {0xA8D9D1535CE3B396, 694, 228}, {0xFB9B7CD9A4A7443C, 720, 236}, - {0xBB764C4CA7A44410, 747, 244}, {0x8BAB8EEFB6409C1A, 774, 252}, - {0xD01FEF10A657842C, 800, 260}, {0x9B10A4E5E9913129, 827, 268}, - {0xE7109BFBA19C0C9D, 853, 276}, {0xAC2820D9623BF429, 880, 284}, - {0x80444B5E7AA7CF85, 907, 292}, {0xBF21E44003ACDD2D, 933, 300}, - {0x8E679C2F5E44FF8F, 960, 308}, {0xD433179D9C8CB841, 986, 316}, - {0x9E19DB92B4E31BA9, 1013, 324}, - }}; - - // This computation gives exactly the same results for k as - // k = ceil((kAlpha - e - 1) * 0.30102999566398114) - // for |e| <= 1500, but doesn't require floating-point operations. - // NB: log_10(2) ~= 78913 / 2^18 - const int f = kAlpha - e - 1; - const int k = (f * 78913) / (1 << 18) + static_cast(f > 0); - - const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / - kCachedPowersDecStep; - - const cached_power cached = kCachedPowers[static_cast(index)]; - - return cached; +inline compute_mul_result compute_mul(std::uint64_t u, uint128 c) noexcept { + const uint128 r = umul192_upper128(u, c); + return {r.high, r.low == 0}; } -/*! -For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. -For n == 0, returns 1 and sets pow10 := 1. -*/ -inline int find_largest_pow10(const std::uint32_t n, std::uint32_t &pow10) { - // LCOV_EXCL_START - if (n >= 1000000000) { - pow10 = 1000000000; - return 10; - } - // LCOV_EXCL_STOP - else if (n >= 100000000) { - pow10 = 100000000; - return 9; - } else if (n >= 10000000) { - pow10 = 10000000; - return 8; - } else if (n >= 1000000) { - pow10 = 1000000; - return 7; - } else if (n >= 100000) { - pow10 = 100000; - return 6; - } else if (n >= 10000) { - pow10 = 10000; - return 5; - } else if (n >= 1000) { - pow10 = 1000; - return 4; - } else if (n >= 100) { - pow10 = 100; - return 3; - } else if (n >= 10) { - pow10 = 10; - return 2; - } else { - pow10 = 1; - return 1; - } +inline std::uint64_t compute_delta(uint128 c, int beta) noexcept { + return c.high >> (total_bits - 1 - beta); } -inline void grisu2_round(char *buf, int len, std::uint64_t dist, - std::uint64_t delta, std::uint64_t rest, - std::uint64_t ten_k) { - - // <--------------------------- delta ----> - // <---- dist ---------> - // --------------[------------------+-------------------]-------------- - // M- w M+ - // - // ten_k - // <------> - // <---- rest ----> - // --------------[------------------+----+--------------]-------------- - // w V - // = buf * 10^k - // - // ten_k represents a unit-in-the-last-place in the decimal representation - // stored in buf. - // Decrement buf by ten_k while this takes buf closer to w. - - // The tests are written in this order to avoid overflow in unsigned - // integer arithmetic. +inline compute_mul_parity_result compute_mul_parity(std::uint64_t two_f, + uint128 c, int beta) noexcept { + // beta is always in [1, 63] here. + const uint128 r = umul192_lower128(two_f, c); + return {((r.high >> (64 - beta)) & 1) != 0, + ((r.high << beta) | (r.low >> (64 - beta))) == 0}; +} - while (rest < dist && delta - rest >= ten_k && - (rest + ten_k < dist || dist - rest > rest + ten_k - dist)) { - buf[len - 1]--; - rest += ten_k; - } +inline std::uint64_t +compute_left_endpoint_for_shorter_interval_case(uint128 c, int beta) noexcept { + return (c.high - (c.high >> (significand_bits + 2))) >> + (total_bits - significand_bits - 1 - beta); } -/*! -Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. -M- and M+ must be normalized and share the same exponent -60 <= e <= -32. -*/ -inline void grisu2_digit_gen(char *buffer, int &length, int &decimal_exponent, - diyfp M_minus, diyfp w, diyfp M_plus) { - static_assert(kAlpha >= -60, "internal error"); - static_assert(kGamma <= -32, "internal error"); +inline std::uint64_t +compute_right_endpoint_for_shorter_interval_case(uint128 c, int beta) noexcept { + return (c.high + (c.high >> (significand_bits + 1))) >> + (total_bits - significand_bits - 1 - beta); +} - // Generates the digits (and the exponent) of a decimal floating-point - // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's - // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= - // gamma. - // - // <--------------------------- delta ----> - // <---- dist ---------> - // --------------[------------------+-------------------]-------------- - // M- w M+ - // - // Grisu2 generates the digits of M+ from left to right and stops as soon as - // V is in [M-,M+]. +inline std::uint64_t +compute_round_up_for_shorter_interval_case(uint128 c, int beta) noexcept { + return ((c.high >> (total_bits - significand_bits - 2 - beta)) + 1) / 2; +} - std::uint64_t delta = - diyfp::sub(M_plus, M_minus) - .f; // (significand of (M+ - M-), implicit exponent is e) - std::uint64_t dist = - diyfp::sub(M_plus, w) - .f; // (significand of (M+ - w ), implicit exponent is e) +// floor(n / 10) for the shorter-interval right endpoint (n bounded so the +// single multiply below is exact). +inline std::uint64_t divide_by_pow10_1(std::uint64_t n) noexcept { + return umul128_upper64(n, std::uint64_t(1844674407370955162ull)); +} - // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): - // - // M+ = f * 2^e - // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e - // = ((p1 ) * 2^-e + (p2 )) * 2^e - // = p1 + p2 * 2^e +// floor(n / 1000) for the larger-divisor step (n bounded as above). +inline std::uint64_t divide_by_pow10_3(std::uint64_t n) noexcept { + return umul128_upper64(n, std::uint64_t(4722366482869645214ull)) >> 8; +} - const diyfp one(std::uint64_t{1} << -M_plus.e, M_plus.e); +// Returns whether n is divisible by 10^kappa (= 100) and divides n by it. +inline bool check_divisibility_and_divide_by_pow10_kappa(std::uint64_t &n) noexcept { + // magic number for division by 100 (kappa == 2). + const std::uint32_t prod = std::uint32_t(n) * std::uint32_t(656); + const bool result = (prod & 0xffffu) < 656u; + n = std::uint64_t(prod >> 16); + return result; +} - auto p1 = static_cast( - M_plus.f >> - -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) - std::uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e +// Strip trailing decimal zeros from significand, bumping exponent accordingly. +// Branchless search; constants from the Dragonbox reference. +inline void remove_trailing_zeros(std::uint64_t &significand, int &exponent) noexcept { + std::uint64_t r = rotr64(significand * std::uint64_t(28999941890838049ull), 8); + bool b = r < std::uint64_t(184467440738ull); + int s = b ? 1 : 0; + significand = b ? r : significand; - // 1) - // - // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] + r = rotr64(significand * std::uint64_t(182622766329724561ull), 4); + b = r < std::uint64_t(1844674407370956ull); + s = s * 2 + (b ? 1 : 0); + significand = b ? r : significand; - std::uint32_t pow10; - const int k = find_largest_pow10(p1, pow10); + r = rotr64(significand * std::uint64_t(10330176681277348905ull), 2); + b = r < std::uint64_t(184467440737095517ull); + s = s * 2 + (b ? 1 : 0); + significand = b ? r : significand; - // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) - // - // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) - // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) - // - // M+ = p1 + p2 * 2^e - // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e - // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e - // = d[k-1] * 10^(k-1) + ( rest) * 2^e - // - // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) - // - // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] - // - // but stop as soon as - // - // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e + r = rotr64(significand * std::uint64_t(14757395258967641293ull), 1); + b = r < std::uint64_t(1844674407370955162ull); + s = s * 2 + (b ? 1 : 0); + significand = b ? r : significand; - int n = k; - while (n > 0) { - // Invariants: - // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) - // pow10 = 10^(n-1) <= p1 < 10^n - // - const std::uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) - const std::uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) - // - // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e - // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) - // - buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d - // - // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) - // - p1 = r; - n--; - // - // M+ = buffer * 10^n + (p1 + p2 * 2^e) - // pow10 = 10^n - // + exponent += s; +} - // Now check if enough digits have been generated. - // Compute - // - // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e - // - // Note: - // Since rest and delta share the same exponent e, it suffices to - // compare the significands. - const std::uint64_t rest = (std::uint64_t{p1} << -one.e) + p2; - if (rest <= delta) { - // V = buffer * 10^n, with M- <= V <= M+. +// Dragonbox core: shortest (significand, exponent) such that +// value == significand * 10^exponent +// for a finite, positive, non-zero binary64 value, decomposed into its raw +// significand bits and biased exponent bits. +struct decimal_fp { + std::uint64_t significand; + int exponent; +}; - decimal_exponent += n; +inline decimal_fp to_decimal(std::uint64_t binary_significand, + int binary_exponent) noexcept { + const bool is_even = (binary_significand % 2 == 0); + std::uint64_t two_fc = binary_significand * 2; + + // Is the input a normal number? + if (binary_exponent != 0) { + binary_exponent += exponent_bias - significand_bits; + + // Shorter interval case; proceed like Schubfach. + if (two_fc == 0) { + const int minus_k = + floor_log10_pow2_minus_log10_4_over_3(binary_exponent); + const int beta = binary_exponent + floor_log2_pow10(-minus_k); + const uint128 c = cache[-minus_k - cache_min_k]; + + std::uint64_t xi = + compute_left_endpoint_for_shorter_interval_case(c, beta); + const std::uint64_t zi = + compute_right_endpoint_for_shorter_interval_case(c, beta); + + // If the left endpoint is not an integer, increase it. (Both endpoints + // are always included since the significand is even.) + if (!(binary_exponent >= + case_shorter_interval_left_endpoint_lower_threshold && + binary_exponent <= + case_shorter_interval_left_endpoint_upper_threshold)) { + ++xi; + } - // We may now just stop. But instead look if the buffer could be - // decremented to bring V closer to w. - // - // pow10 = 10^n is now 1 ulp in the decimal representation V. - // The rounding procedure works with diyfp's with an implicit - // exponent of e. - // - // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e - // - const std::uint64_t ten_n = std::uint64_t{pow10} << -one.e; - grisu2_round(buffer, length, dist, delta, rest, ten_n); + // Try the bigger divisor. + std::uint64_t decimal_significand = divide_by_pow10_1(zi); + if (decimal_significand * 10 >= xi) { + int decimal_exponent = minus_k + 1; + remove_trailing_zeros(decimal_significand, decimal_exponent); + return {decimal_significand, decimal_exponent}; + } - return; + // Otherwise, compute the round-up of y. + decimal_significand = + compute_round_up_for_shorter_interval_case(c, beta); + // On a tie, choose the even one. + if ((decimal_significand % 2 != 0) && + binary_exponent >= shorter_interval_tie_lower_threshold && + binary_exponent <= shorter_interval_tie_upper_threshold) { + --decimal_significand; + } else if (decimal_significand < xi) { + ++decimal_significand; + } + return {decimal_significand, minus_k}; } - pow10 /= 10; - // - // pow10 = 10^(n-1) <= p1 < 10^n - // Invariants restored. - } - - // 2) - // - // The digits of the integral part have been generated: - // - // M+ = d[k-1]...d[1]d[0] + p2 * 2^e - // = buffer + p2 * 2^e - // - // Now generate the digits of the fractional part p2 * 2^e. - // - // Note: - // No decimal point is generated: the exponent is adjusted instead. - // - // p2 actually represents the fraction - // - // p2 * 2^e - // = p2 / 2^-e - // = d[-1] / 10^1 + d[-2] / 10^2 + ... - // - // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) - // - // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m - // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) - // - // using - // - // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) - // = ( d) * 2^-e + ( r) - // - // or - // 10^m * p2 * 2^e = d + r * 2^e - // - // i.e. - // - // M+ = buffer + p2 * 2^e - // = buffer + 10^-m * (d + r * 2^e) - // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e - // - // and stop as soon as 10^-m * r * 2^e <= delta * 2^e - - int m = 0; - for (;;) { - // Invariant: - // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) - // * 2^e - // = buffer * 10^-m + 10^-m * (p2 ) - // * 2^e = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e = - // buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + - // (10*p2 mod 2^-e)) * 2^e - // - p2 *= 10; - const std::uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e - const std::uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e - // - // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e - // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) - // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e - // - buffer[length++] = static_cast('0' + d); // buffer := buffer * 10 + d - // - // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e - // - p2 = r; - m++; - // - // M+ = buffer * 10^-m + 10^-m * p2 * 2^e - // Invariant restored. - - // Check if enough digits have been generated. - // - // 10^-m * p2 * 2^e <= delta * 2^e - // p2 * 2^e <= 10^m * delta * 2^e - // p2 <= 10^m * delta - delta *= 10; - dist *= 10; - if (p2 <= delta) { + // Normal interval case. + two_fc |= (std::uint64_t(1) << (significand_bits + 1)); + } else { + // Subnormal number: normal interval case. + binary_exponent = min_exponent - significand_bits; + } + + // Step 1: Schubfach multiplier calculation. + const int minus_k = floor_log10_pow2(binary_exponent) - kappa; + const uint128 c = cache[-minus_k - cache_min_k]; + const int beta = binary_exponent + floor_log2_pow10(-minus_k); + + const std::uint64_t deltai = compute_delta(c, beta); + const compute_mul_result z_result = + compute_mul((two_fc | 1) << beta, c); + + // Step 2: Try larger divisor; remove trailing zeros if necessary. + std::uint64_t decimal_significand = divide_by_pow10_3(z_result.integer_part); + std::uint64_t r = + z_result.integer_part - std::uint64_t(big_divisor) * decimal_significand; + + do { + if (r < deltai) { + // Exclude the right endpoint if necessary. + if ((r | std::uint64_t(!z_result.is_integer) | std::uint64_t(is_even)) == + 0) { + --decimal_significand; + r = big_divisor; + break; + } + } else if (r > deltai) { break; + } else { + // r == deltai; compare fractional parts. + const compute_mul_parity_result x_result = + compute_mul_parity(two_fc - 1, c, beta); + if (!(x_result.parity | (x_result.is_integer & is_even))) { + break; + } } - } - - // V = buffer * 10^-m, with M- <= V <= M+. - - decimal_exponent -= m; - - // 1 ulp in the decimal representation is now 10^-m. - // Since delta and dist are now scaled by 10^m, we need to do the - // same with ulp in order to keep the units in sync. - // - // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e - // - const std::uint64_t ten_m = one.f; - grisu2_round(buffer, length, dist, delta, p2, ten_m); - // By construction this algorithm generates the shortest possible decimal - // number (Loitsch, Theorem 6.2) which rounds back to w. - // For an input number of precision p, at least - // - // N = 1 + ceil(p * log_10(2)) - // - // decimal digits are sufficient to identify all binary floating-point - // numbers (Matula, "In-and-Out conversions"). - // This implies that the algorithm does not produce more than N decimal - // digits. - // - // N = 17 for p = 53 (IEEE double precision) - // N = 9 for p = 24 (IEEE single precision) -} + int decimal_exponent = minus_k + kappa + 1; + remove_trailing_zeros(decimal_significand, decimal_exponent); + return {decimal_significand, decimal_exponent}; + } while (false); -/*! -v = buf * 10^decimal_exponent -len is the length of the buffer (number of decimal digits) -The buffer must be large enough, i.e. >= max_digits10. -*/ -inline void grisu2(char *buf, int &len, int &decimal_exponent, diyfp m_minus, - diyfp v, diyfp m_plus) { - - // --------(-----------------------+-----------------------)-------- (A) - // m- v m+ - // - // --------------------(-----------+-----------------------)-------- (B) - // m- v m+ - // - // First scale v (and m- and m+) such that the exponent is in the range - // [alpha, gamma]. + // Step 3: Find the significand with the smaller divisor. + decimal_significand *= 10; - const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); + std::uint64_t dist = r - (deltai / 2) + (small_divisor / 2); + const bool approx_y_parity = ((dist ^ (small_divisor / 2)) & 1) != 0; - const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k + const bool divisible_by_small_divisor = + check_divisibility_and_divide_by_pow10_kappa(dist); - // The exponent of the products is = v.e + c_minus_k.e + q and is in the range - // [alpha,gamma] - const diyfp w = diyfp::mul(v, c_minus_k); - const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); - const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); + decimal_significand += dist; - // ----(---+---)---------------(---+---)---------------(---+---)---- - // w- w w+ - // = c*m- = c*v = c*m+ - // - // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and - // w+ are now off by a small amount. - // In fact: - // - // w - v * 10^k < 1 ulp - // - // To account for this inaccuracy, add resp. subtract 1 ulp. - // - // --------+---[---------------(---+---)---------------]---+-------- - // w- M- w M+ w+ - // - // Now any number in [M-, M+] (bounds included) will round to w when input, - // regardless of how the input rounding algorithm breaks ties. - // - // And digit_gen generates the shortest possible such number in [M-, M+]. - // Note that this does not mean that Grisu2 always generates the shortest - // possible number in the interval (m-, m+). - const diyfp M_minus(w_minus.f + 1, w_minus.e); - const diyfp M_plus(w_plus.f - 1, w_plus.e); - - decimal_exponent = -cached.k; // = -(-k) = k + if (divisible_by_small_divisor) { + const compute_mul_parity_result y_result = + compute_mul_parity(two_fc, c, beta); + if (y_result.parity != approx_y_parity) { + --decimal_significand; + } else if ((decimal_significand % 2) != 0 && y_result.is_integer) { + // On a tie (y is an integer), choose the even one. + --decimal_significand; + } + } - grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); + return {decimal_significand, minus_k + kappa}; } /*! -v = buf * 10^decimal_exponent -len is the length of the buffer (number of decimal digits) -The buffer must be large enough, i.e. >= max_digits10. +Fills buf with the shortest decimal digits of 'value' (which must be finite, +positive and non-zero), sets len to the number of digits, and decimal_exponent +so that value == (buf, interpreted as an integer) * 10^decimal_exponent. +This mirrors the contract of the previous grisu2() entry point. */ -template -void grisu2(char *buf, int &len, int &decimal_exponent, FloatType value) { - static_assert(diyfp::kPrecision >= std::numeric_limits::digits + 3, - "internal error: not enough precision"); - - // If the neighbors (and boundaries) of 'value' are always computed for - // double-precision numbers, all float's can be recovered using strtod (and - // strtof). However, the resulting decimal representations are not exactly - // "short". - // - // The documentation for 'std::to_chars' - // (https://en.cppreference.com/w/cpp/utility/to_chars) says "value is - // converted to a string as if by std::sprintf in the default ("C") locale" - // and since sprintf promotes float's to double's, I think this is exactly - // what 'std::to_chars' does. On the other hand, the documentation for - // 'std::to_chars' requires that "parsing the representation using the - // corresponding std::from_chars function recovers value exactly". That - // indicates that single precision floating-point numbers should be recovered - // using 'std::strtof'. - // - // NB: If the neighbors are computed for single-precision numbers, there is a - // single float - // (7.0385307e-26f) which can't be recovered using strtod. The resulting - // double precision value is off by 1 ulp. -#if 0 - const boundaries w = compute_boundaries(static_cast(value)); -#else - const boundaries w = compute_boundaries(value); -#endif - - grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); +inline void dragonbox(char *buf, int &len, int &decimal_exponent, + double value) { + std::uint64_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + const std::uint64_t binary_significand = + bits & ((std::uint64_t(1) << significand_bits) - 1); + const int binary_exponent = + int((bits >> significand_bits) & 0x7ff); + + const decimal_fp dec = to_decimal(binary_significand, binary_exponent); + + // Convert the decimal significand to digits: + // 1) Proceed 2 digits at a time (s % 100) via a 00..99 lookup table + // (see Alexandrescu, "Three Optimization Tips for C++", 2012), + // 2) Digits come out least-significant first, writing them back-to-front + // with p = tmp + sizeof(tmp); to avoid reversal pass + // 3) Proceed remaining digits after loop to avoid branchs inside it + // 4) memcpy digits to char* buf (inside function input) + static const char digits2[201] = + "0001020304050607080910111213141516171819" + "2021222324252627282930313233343536373839" + "4041424344454647484950515253545556575859" + "6061626364656667686970717273747576777879" + "8081828384858687888990919293949596979899"; + // Digit area is 24; +16 padding lets us always memcpy 16 (+1) bytes with a + // compile-time size so the compiler inlines (no libc size-class branches). + // Callers must provide to_chars_buffer_size (40) bytes for the same reason. + constexpr int digit_area = 24; + char tmp[digit_area + 16]; + char *p = tmp + digit_area; // write backward + std::uint64_t s = dec.significand; + while (s >= 100) { + const std::uint32_t idx = static_cast(s % 100) * 2; + s /= 100; + p -= 2; + p[0] = digits2[idx]; + p[1] = digits2[idx + 1]; + } + if (s >= 10) { + const std::uint32_t idx = static_cast(s) * 2; + p -= 2; + p[0] = digits2[idx]; + p[1] = digits2[idx + 1]; + } else { + *--p = static_cast('0' + s); + } + // Fixed-size copy: double has at most 17 significant digits. + std::memcpy(buf, p, 16); + buf[16] = p[16]; + len = static_cast(tmp + digit_area - p); + decimal_exponent = dec.exponent; } /*! @@ -4111,11 +4625,16 @@ inline char *format_buffer(char *buf, int len, int decimal_exponent, // k is the length of the buffer (number of decimal digits) // n is the position of the decimal point relative to the start of the buffer. + // All mem* sizes below are compile-time constants so the compiler inlines + // them as plain loads/stores. That requires over-writing past the logical + // string length; callers must reserve to_chars_buffer_size (40) bytes. + // Logical output is still bounded by ~24 characters; only the returned + // pointer reflects the true length. + if (k <= n && n <= max_exp) { // digits[000] // len <= max_exp + 2 - - std::memset(buf + k, '0', static_cast(n) - static_cast(k)); + std::memset(buf + k, '0', 16); // Make it look like a floating-point number (#362, #378) buf[n + 0] = '.'; buf[n + 1] = '0'; @@ -4125,690 +4644,5307 @@ inline char *format_buffer(char *buf, int len, int decimal_exponent, if (0 < n && n <= max_exp) { // dig.its // len <= max_digits10 + 1 - std::memmove(buf + (static_cast(n) + 1), buf + n, - static_cast(k) - static_cast(n)); + // Shift the fractional digits one place right via a temp (overlap). + char shifted[16]; + std::memcpy(shifted, buf + static_cast(n), 16); + std::memcpy(buf + (static_cast(n) + 1), shifted, 16); buf[n] = '.'; return buf + (static_cast(k) + 1U); } - if (min_exp < n && n <= 0) { - // 0.[000]digits - // len <= 2 + (-min_exp - 1) + max_digits10 + if (min_exp < n && n <= 0) { + // 0.[000]digits + // With kMinExp = -4, n is in {-3,-2,-1,0}, so pad = -n is 0..3. + // len <= 2 + (-min_exp - 1) + max_digits10 + char digits[17]; + std::memcpy(digits, buf, 17); + const size_t pad = static_cast(-n); // 0..3 + buf[0] = '0'; + buf[1] = '.'; + // Fixed upper bound on leading zeros; only the first `pad` matter. + std::memset(buf + 2, '0', 4); + std::memcpy(buf + 2 + pad, digits, 17); + return buf + (2U + pad + static_cast(k)); + } + + if (k == 1) { + // dE+123 + // len <= 1 + 5 + buf += 1; + } else { + // d.igitsE+123 + // len <= max_digits10 + 1 + 5 + // k-1 <= 16 for double; fixed-size shift via temp (overlap). + char shifted[16]; + std::memcpy(shifted, buf + 1, 16); + std::memcpy(buf + 2, shifted, 16); + buf[1] = '.'; + buf += 1 + static_cast(k); + } + + *buf++ = 'e'; + return append_exponent(buf, n - 1); +} + +} // NS dtoa_impl + +/*! +The format of the resulting decimal representation is similar to printf's %g +format. Returns an iterator pointing past-the-end of the decimal representation. +@note The input number must be finite, i.e. NaN's and Inf's are not supported. +@note The buffer must have at least to_chars_buffer_size (40) writable bytes. + Only ~24 characters are ever part of the logical result, but fixed-size + 16/17-byte mem* over-writes require the extra scratch for safety. +@note The result is NOT null-terminated. +*/ +char *to_chars(char *first, const char *last, double value) { + static_cast(last); // maybe unused - fix warning + bool negative = std::signbit(value); + if (negative) { + value = -value; + *first++ = '-'; + } + + if (value == 0) // +-0 + { + *first++ = '0'; + // Make it look like a floating-point number (#362, #378) + *first++ = '.'; + *first++ = '0'; + return first; + } + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e. the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + dtoa_impl::dragonbox(first, len, decimal_exponent, value); + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + constexpr int kMaxExp = std::numeric_limits::digits10; + + return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, + kMaxExp); +} +} // NS internal +} // NS simdjson + +#endif // SIMDJSON_SRC_TO_CHARS_CPP + + +/* end file to_chars.cpp */ +/* including from_chars.cpp: #include */ +/* begin file from_chars.cpp */ +#ifndef SIMDJSON_SRC_FROM_CHARS_CPP +#define SIMDJSON_SRC_FROM_CHARS_CPP + +/* skipped duplicate #include */ + +/* including simdjson/internal/fast_float.h: #include "simdjson/internal/fast_float.h" */ +/* begin file simdjson/internal/fast_float.h */ +// Vendored from fast_float v8.2.10, generated by tools/vendor_fast_float.sh. +// Do not edit by hand; re-run the script to update. +// +// https://github.com/fastfloat/fast_float +// Licensed under Apache-2.0 OR MIT OR BSL-1.0, at your option. +// +// simdjson uses this for two things that its own number parser cannot do: +// * the slow path for numbers with more than 19 significant digits, where +// fast_float's bigint comparison is several times quicker than the +// Wuffs-derived decimal shifting it replaced (see src/from_chars.cpp), and +// * correctly rounded parsing inside a constant expression, which the runtime +// path cannot offer because it relies on memcpy and __uint128_t (see +// compile_time_json-inl.h). +// +// Two edits are applied by the script. Every fast_float name is rewritten so +// that this copy cannot collide with a copy of fast_float that the surrounding +// program includes for itself: namespace fast_float -> simdjson_fast_float, +// FASTFLOAT_* -> SIMDJSON_FASTFLOAT_*, fastfloat_* -> simdjson_fastfloat_*. And +// the accented letters in the attribution comments below are folded to ASCII, +// to keep the tree ASCII-only; no disrespect to the people named is intended. +// simdjson_fast_float by Daniel Lemire +// simdjson_fast_float by Joao Paulo Magalhaes +// +// +// with contributions from Eugene Golushkov +// with contributions from Maksim Kita +// with contributions from Marcin Wojdyr +// with contributions from Neal Richardson +// with contributions from Tim Paine +// with contributions from Fabio Pellacini +// with contributions from Lenard Szolnoki +// with contributions from Jan Pharago +// with contributions from Maya Warrier +// with contributions from Taha Khokhar +// with contributions from Anders Dalvander +// +// +// Licensed under the Apache License, Version 2.0, or the +// MIT License or the Boost License. This file may not be copied, +// modified, or distributed except according to those terms. +// +// MIT License Notice +// +// MIT License +// +// Copyright (c) 2021 The simdjson_fast_float authors +// +// Permission is hereby granted, free of charge, to any +// person obtaining a copy of this software and associated +// documentation files (the "Software"), to deal in the +// Software without restriction, including without +// limitation the rights to use, copy, modify, merge, +// publish, distribute, sublicense, and/or sell copies of +// the Software, and to permit persons to whom the Software +// is furnished to do so, subject to the following +// conditions: +// +// The above copyright notice and this permission notice +// shall be included in all copies or substantial portions +// of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +// ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +// TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +// PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +// SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +// IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// +// Apache License (Version 2.0) Notice +// +// Copyright 2021 The simdjson_fast_float authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// +// BOOST License Notice +// +// Boost Software License - Version 1.0 - August 17th, 2003 +// +// Permission is hereby granted, free of charge, to any person or organization +// obtaining a copy of the software and accompanying documentation covered by +// this license (the "Software") to use, reproduce, display, distribute, +// execute, and transmit the Software, and to prepare derivative works of the +// Software, and to permit third-parties to whom the Software is furnished to +// do so, all subject to the following: +// +// The copyright notices in the Software and this entire statement, including +// the above license grant, this restriction and the following disclaimer, +// must be included in all copies of the Software, in whole or in part, and +// all derivative works of the Software, unless such copies or derivative +// works are solely in the form of machine-executable object code generated by +// a source language processor. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +// SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +// FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + +#ifndef SIMDJSON_FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H +#define SIMDJSON_FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H + +#ifdef __has_include +#if __has_include() +#include +#endif +#endif + +// Testing for https://wg21.link/N3652, adopted in C++14 +#if defined(__cpp_constexpr) && __cpp_constexpr >= 201304 +#define SIMDJSON_FASTFLOAT_CONSTEXPR14 constexpr +#else +#define SIMDJSON_FASTFLOAT_CONSTEXPR14 +#endif + +#if defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L +#define SIMDJSON_FASTFLOAT_HAS_BIT_CAST 1 +#else +#define SIMDJSON_FASTFLOAT_HAS_BIT_CAST 0 +#endif + +#if defined(__cpp_lib_is_constant_evaluated) && \ + __cpp_lib_is_constant_evaluated >= 201811L +#define SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED 1 +#else +#define SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED 0 +#endif + +#if defined(__cpp_if_constexpr) && __cpp_if_constexpr >= 201606L +#define SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(x) if constexpr (x) +#else +#define SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(x) if (x) +#endif + +// Testing for relevant C++20 constexpr library features +#if SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED && SIMDJSON_FASTFLOAT_HAS_BIT_CAST && \ + defined(__cpp_lib_constexpr_algorithms) && \ + __cpp_lib_constexpr_algorithms >= 201806L /*For std::copy and std::fill*/ +#define SIMDJSON_FASTFLOAT_CONSTEXPR20 constexpr +#define SIMDJSON_FASTFLOAT_IS_CONSTEXPR 1 +#else +#define SIMDJSON_FASTFLOAT_CONSTEXPR20 +#define SIMDJSON_FASTFLOAT_IS_CONSTEXPR 0 +#endif + +#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) +#define SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE 0 +#else +#define SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE 1 +#endif + +#endif // SIMDJSON_FASTFLOAT_CONSTEXPR_FEATURE_DETECT_H + +#ifndef SIMDJSON_FASTFLOAT_FLOAT_COMMON_H +#define SIMDJSON_FASTFLOAT_FLOAT_COMMON_H + +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __has_include +#if __has_include() && (__cplusplus > 202002L || (defined(_MSVC_LANG) && (_MSVC_LANG > 202002L))) +#include +#endif +#endif + +#define SIMDJSON_FASTFLOAT_VERSION_MAJOR 8 +#define SIMDJSON_FASTFLOAT_VERSION_MINOR 2 +#define SIMDJSON_FASTFLOAT_VERSION_PATCH 10 + +#define SIMDJSON_FASTFLOAT_STRINGIZE_IMPL(x) #x +#define SIMDJSON_FASTFLOAT_STRINGIZE(x) SIMDJSON_FASTFLOAT_STRINGIZE_IMPL(x) + +#define SIMDJSON_FASTFLOAT_VERSION_STR \ + SIMDJSON_FASTFLOAT_STRINGIZE(SIMDJSON_FASTFLOAT_VERSION_MAJOR) \ + "." SIMDJSON_FASTFLOAT_STRINGIZE(SIMDJSON_FASTFLOAT_VERSION_MINOR) "." SIMDJSON_FASTFLOAT_STRINGIZE( \ + SIMDJSON_FASTFLOAT_VERSION_PATCH) + +#define SIMDJSON_FASTFLOAT_VERSION \ + (SIMDJSON_FASTFLOAT_VERSION_MAJOR * 10000 + SIMDJSON_FASTFLOAT_VERSION_MINOR * 100 + \ + SIMDJSON_FASTFLOAT_VERSION_PATCH) + +namespace simdjson_fast_float { + +enum class chars_format : uint64_t; + +namespace detail { +constexpr chars_format basic_json_fmt = chars_format(1 << 5); +constexpr chars_format basic_fortran_fmt = chars_format(1 << 6); +} // namespace detail + +enum class chars_format : uint64_t { + scientific = 1 << 0, + fixed = 1 << 2, + hex = 1 << 3, + no_infnan = 1 << 4, + // RFC 8259: https://datatracker.ietf.org/doc/html/rfc8259#section-6 + json = uint64_t(detail::basic_json_fmt) | fixed | scientific | no_infnan, + // Extension of RFC 8259 where, e.g., "inf" and "nan" are allowed. + json_or_infnan = uint64_t(detail::basic_json_fmt) | fixed | scientific, + fortran = uint64_t(detail::basic_fortran_fmt) | fixed | scientific, + general = fixed | scientific, + allow_leading_plus = 1 << 7, + skip_white_space = 1 << 8, +}; + +template struct from_chars_result_t { + UC const *ptr; + std::errc ec; + + // https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2497r0.html + constexpr explicit operator bool() const noexcept { + return ec == std::errc(); + } +}; + +using from_chars_result = from_chars_result_t; + +template struct parse_options_t { + constexpr explicit parse_options_t(chars_format fmt = chars_format::general, + UC dot = UC('.'), int b = 10) + : format(fmt), decimal_point(dot), base(b) {} + + /** Which number formats are accepted */ + chars_format format; + /** The character used as decimal point */ + UC decimal_point; + /** The base used for integers */ + int base; +}; + +using parse_options = parse_options_t; + +} // namespace simdjson_fast_float + +#if SIMDJSON_FASTFLOAT_HAS_BIT_CAST +#include +#endif + +#if (defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(__amd64) || defined(__aarch64__) || defined(_M_ARM64) || \ + defined(__MINGW64__) || defined(__s390x__) || \ + (defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \ + defined(__PPC64LE__)) || \ + defined(__loongarch64) || (defined(__riscv) && __riscv_xlen == 64)) +#define SIMDJSON_FASTFLOAT_64BIT 1 +#elif (defined(__i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__arm__) || defined(_M_ARM) || defined(__ppc__) || \ + defined(__MINGW32__) || defined(__EMSCRIPTEN__) || \ + (defined(__riscv) && __riscv_xlen == 32)) +#define SIMDJSON_FASTFLOAT_32BIT 1 +#else + // Need to check incrementally, since SIZE_MAX is a size_t, avoid overflow. +// We can never tell the register width, but the SIZE_MAX is a good +// approximation. UINTPTR_MAX and INTPTR_MAX are optional, so avoid them for max +// portability. +#if SIZE_MAX == 0xffff +#error Unknown platform (16-bit, unsupported) +#elif SIZE_MAX == 0xffffffff +#define SIMDJSON_FASTFLOAT_32BIT 1 +#elif SIZE_MAX == 0xffffffffffffffff +#define SIMDJSON_FASTFLOAT_64BIT 1 +#else +#error Unknown platform (not 32-bit, not 64-bit?) +#endif +#endif + +#if ((defined(_WIN32) || defined(_WIN64)) && !defined(__clang__)) || \ + (defined(_M_ARM64) && !defined(__MINGW32__)) +#include +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +#define SIMDJSON_FASTFLOAT_VISUAL_STUDIO 1 +#endif + +#if defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#elif defined _WIN32 +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 +#else +#if defined(__APPLE__) || defined(__FreeBSD__) +#include +#elif defined(sun) || defined(__sun) +#include +#elif defined(__MVS__) +#include +#else +#ifdef __has_include +#if __has_include() +#include +#endif //__has_include() +#endif //__has_include +#endif +# +#ifndef __BYTE_ORDER__ +// safe choice +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 +#endif +# +#ifndef __ORDER_LITTLE_ENDIAN__ +// safe choice +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 +#endif +# +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 0 +#else +#define SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN 1 +#endif +#endif + +#if defined(__SSE2__) || (defined(SIMDJSON_FASTFLOAT_VISUAL_STUDIO) && \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2))) +#define SIMDJSON_FASTFLOAT_SSE2 1 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#define SIMDJSON_FASTFLOAT_NEON 1 +#endif + +#if defined(SIMDJSON_FASTFLOAT_SSE2) || defined(SIMDJSON_FASTFLOAT_NEON) +#define SIMDJSON_FASTFLOAT_HAS_SIMD 1 +#endif + +#if defined(__GNUC__) +// disable -Wcast-align=strict (GCC only) +#define SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wcast-align\"") +#else +#define SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS +#endif + +#if defined(__GNUC__) +#define SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS _Pragma("GCC diagnostic pop") +#else +#define SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +#endif + +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#define simdjson_fastfloat_really_inline __forceinline +#else +#define simdjson_fastfloat_really_inline inline __attribute__((always_inline)) +#endif + +// Branch-probability hint marking the rare slow-path branches as cold, so the +// optimizer keeps the out-of-line slow-path re-parse off the hot path (and does +// not duplicate the force-inlined hot scanner into the caller, which bloated +// the hot frame and hurt ILP on some targets). Used at the call site as +// if simdjson_fastfloat_unlikely(cond) { ... } +// (the macro supplies the parentheses). It expands to the standard [[unlikely]] +// attribute when supported, otherwise to __builtin_expect on GCC/Clang, or +// to a no-op elsewhere (e.g. pre-C++20 MSVC, which has no equivalent hint). +#ifdef __has_cpp_attribute +#if __has_cpp_attribute(unlikely) >= 201803L +// g++-9 hits hits this branch, but then fails to compile +// [[unlikely]]. This happens only with g++-9. +#if !defined(__GNUC__) || (__GNUC__ != 9) +#define SIMDJSON_FASTFLOAT_USE_UNLIKELY_ATTR +#endif +#endif +#endif + +#ifdef SIMDJSON_FASTFLOAT_USE_UNLIKELY_ATTR +#define simdjson_fastfloat_unlikely(x) (x) [[unlikely]] +#elif defined(__GNUC__) || defined(__clang__) +#define simdjson_fastfloat_unlikely(x) (__builtin_expect(!!(x), 0)) +#else +#define simdjson_fastfloat_unlikely(x) (x) +#endif + +#ifndef SIMDJSON_FASTFLOAT_ASSERT +#define SIMDJSON_FASTFLOAT_ASSERT(x) \ + { \ + static_cast(x); \ + } +#endif + +#ifndef SIMDJSON_FASTFLOAT_DEBUG_ASSERT +#define SIMDJSON_FASTFLOAT_DEBUG_ASSERT(x) \ + { \ + static_cast(x); \ + } +#endif + +// rust style `try!()` macro, or `?` operator +#define SIMDJSON_FASTFLOAT_TRY(x) \ + { \ + if (!(x)) \ + return false; \ + } + +#define SIMDJSON_FASTFLOAT_ENABLE_IF(...) \ + typename std::enable_if<(__VA_ARGS__), int>::type + +namespace simdjson_fast_float { + +simdjson_fastfloat_really_inline constexpr bool cpp20_and_in_constexpr() { +#if SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED + return std::is_constant_evaluated(); +#else + return false; +#endif +} + +template +struct is_supported_float_type + : std::integral_constant< + bool, std::is_same::value || std::is_same::value +#ifdef __STDCPP_FLOAT64_T__ + || std::is_same::value +#endif +#ifdef __STDCPP_FLOAT32_T__ + || std::is_same::value +#endif +#ifdef __STDCPP_FLOAT16_T__ + || std::is_same::value +#endif +#ifdef __STDCPP_BFLOAT16_T__ + || std::is_same::value +#endif + > { +}; + +template +using equiv_uint_t = typename std::conditional< + sizeof(T) == 1, uint8_t, + typename std::conditional< + sizeof(T) == 2, uint16_t, + typename std::conditional::type>::type>::type; + +template struct is_supported_integer_type : std::is_integral {}; + +template +struct is_supported_char_type + : std::integral_constant::value || + std::is_same::value || + std::is_same::value || + std::is_same::value +#ifdef __cpp_char8_t + || std::is_same::value +#endif + > { +}; + +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR14 bool +simdjson_fastfloat_strncasecmp3(UC const *actual_mixedcase, + UC const *expected_lowercase) { + uint64_t mask{0}; + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { mask = 0x2020202020202020; } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) { + mask = 0x0020002000200020; + } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { + mask = 0x0000002000000020; + } + else { + return false; + } + + uint64_t val1{0}, val2{0}; + if (cpp20_and_in_constexpr()) { + for (size_t i = 0; i < 3; i++) { + if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { + return false; + } + } + return true; + } else { + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1 || sizeof(UC) == 2) { + ::memcpy(&val1, actual_mixedcase, 3 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase, 3 * sizeof(UC)); + val1 |= mask; + val2 |= mask; + return val1 == val2; + } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { + ::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase, 2 * sizeof(UC)); + val1 |= mask; + if (val1 != val2) { + return false; + } + return (actual_mixedcase[2] | 32) == (expected_lowercase[2]); + } + else { + return false; + } + } +} + +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR14 bool +simdjson_fastfloat_strncasecmp5(UC const *actual_mixedcase, + UC const *expected_lowercase) { + uint64_t mask{0}; + uint64_t val1{0}, val2{0}; + if (cpp20_and_in_constexpr()) { + for (size_t i = 0; i < 5; i++) { + if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { + return false; + } + } + return true; + } else { + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { + mask = 0x2020202020202020; + ::memcpy(&val1, actual_mixedcase, 5 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase, 5 * sizeof(UC)); + val1 |= mask; + val2 |= mask; + return val1 == val2; + } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) { + mask = 0x0020002000200020; + ::memcpy(&val1, actual_mixedcase, 4 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase, 4 * sizeof(UC)); + val1 |= mask; + if (val1 != val2) { + return false; + } + return (actual_mixedcase[4] | 32) == (expected_lowercase[4]); + } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { + mask = 0x0000002000000020; + ::memcpy(&val1, actual_mixedcase, 2 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase, 2 * sizeof(UC)); + val1 |= mask; + if (val1 != val2) { + return false; + } + ::memcpy(&val1, actual_mixedcase + 2, 2 * sizeof(UC)); + ::memcpy(&val2, expected_lowercase + 2, 2 * sizeof(UC)); + val1 |= mask; + if (val1 != val2) { + return false; + } + return (actual_mixedcase[4] | 32) == (expected_lowercase[4]); + } + else { + return false; + } + } +} + +// Compares two ASCII strings in a case insensitive manner. +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR14 bool +simdjson_fastfloat_strncasecmp(UC const *actual_mixedcase, UC const *expected_lowercase, + size_t length) { + uint64_t mask{0}; + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 1) { mask = 0x2020202020202020; } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 2) { + mask = 0x0020002000200020; + } + else SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(sizeof(UC) == 4) { + mask = 0x0000002000000020; + } + else { + return false; + } + + if (cpp20_and_in_constexpr()) { + for (size_t i = 0; i < length; i++) { + if ((actual_mixedcase[i] | 32) != expected_lowercase[i]) { + return false; + } + } + return true; + } else { + uint64_t val1{0}, val2{0}; + size_t sz{8 / (sizeof(UC))}; + for (size_t i = 0; i < length; i += sz) { + val1 = val2 = 0; + sz = sz < (length - i) ? sz : length - i; + ::memcpy(&val1, actual_mixedcase + i, sz * sizeof(UC)); + ::memcpy(&val2, expected_lowercase + i, sz * sizeof(UC)); + val1 |= mask; + val2 |= mask; + if (val1 != val2) { + return false; + } + } + return true; + } +} + +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif + +// a pointer and a length to a contiguous block of memory +template struct span { + T const *ptr; + size_t length; + + constexpr span(T const *_ptr, size_t _length) : ptr(_ptr), length(_length) {} + + constexpr span() : ptr(nullptr), length(0) {} + + constexpr size_t len() const noexcept { return length; } + + SIMDJSON_FASTFLOAT_CONSTEXPR14 const T &operator[](size_t index) const noexcept { + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); + return ptr[index]; + } +}; + +struct value128 { + uint64_t low; + uint64_t high; + + constexpr value128(uint64_t _low, uint64_t _high) : low(_low), high(_high) {} + + constexpr value128() : low(0), high(0) {} +}; + +/* Helper C++14 constexpr generic implementation of leading_zeroes */ +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 int +leading_zeroes_generic(uint64_t input_num, int last_bit = 0) { + if (input_num & uint64_t(0xffffffff00000000)) { + input_num >>= 32; + last_bit |= 32; + } + if (input_num & uint64_t(0xffff0000)) { + input_num >>= 16; + last_bit |= 16; + } + if (input_num & uint64_t(0xff00)) { + input_num >>= 8; + last_bit |= 8; + } + if (input_num & uint64_t(0xf0)) { + input_num >>= 4; + last_bit |= 4; + } + if (input_num & uint64_t(0xc)) { + input_num >>= 2; + last_bit |= 2; + } + if (input_num & uint64_t(0x2)) { /* input_num >>= 1; */ + last_bit |= 1; + } + return 63 - last_bit; +} + +/* result might be undefined when input_num is zero */ +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 int +leading_zeroes(uint64_t input_num) { + assert(input_num > 0); + if (cpp20_and_in_constexpr()) { + return leading_zeroes_generic(input_num); + } +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#if defined(_M_X64) || defined(_M_ARM64) + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + _BitScanReverse64(&leading_zero, input_num); + return static_cast(63 - leading_zero); +#else + return leading_zeroes_generic(input_num); +#endif +#else + return __builtin_clzll(input_num); +#endif +} + +/* Helper C++14 constexpr generic implementation of countr_zero for 32-bit */ +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 int +countr_zero_generic_32(uint32_t input_num) { + if (input_num == 0) { + return 32; + } + int last_bit = 0; + if (!(input_num & 0x0000FFFF)) { + input_num >>= 16; + last_bit |= 16; + } + if (!(input_num & 0x00FF)) { + input_num >>= 8; + last_bit |= 8; + } + if (!(input_num & 0x0F)) { + input_num >>= 4; + last_bit |= 4; + } + if (!(input_num & 0x3)) { + input_num >>= 2; + last_bit |= 2; + } + if (!(input_num & 0x1)) { + last_bit |= 1; + } + return last_bit; +} + +/* count trailing zeroes for 32-bit integers */ +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 int +countr_zero_32(uint32_t input_num) { + if (cpp20_and_in_constexpr()) { + return countr_zero_generic_32(input_num); + } +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO + unsigned long trailing_zero = 0; + if (_BitScanForward(&trailing_zero, input_num)) { + return static_cast(trailing_zero); + } + return 32; +#else + return input_num == 0 ? 32 : __builtin_ctz(input_num); +#endif +} + +// slow emulation routine for 32-bit +simdjson_fastfloat_really_inline constexpr uint64_t emulu(uint32_t x, uint32_t y) { + return x * static_cast(y); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint64_t +umul128_generic(uint64_t ab, uint64_t cd, uint64_t *hi) { + uint64_t ad = + emulu(static_cast(ab >> 32), static_cast(cd)); + uint64_t bd = emulu(static_cast(ab), static_cast(cd)); + uint64_t adbc = + ad + emulu(static_cast(ab), static_cast(cd >> 32)); + uint64_t adbc_carry = static_cast(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = + emulu(static_cast(ab >> 32), static_cast(cd >> 32)) + + (adbc >> 32) + (adbc_carry << 32) + static_cast(lo < bd); + return lo; +} + +#ifdef SIMDJSON_FASTFLOAT_32BIT + +// slow emulation routine for 32-bit +#if !defined(__MINGW64__) +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint64_t _umul128(uint64_t ab, + uint64_t cd, + uint64_t *hi) { + return umul128_generic(ab, cd, hi); +} +#endif // !__MINGW64__ + +#endif // SIMDJSON_FASTFLOAT_32BIT + +// compute 64-bit a*b +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 value128 +full_multiplication(uint64_t a, uint64_t b) { + if (cpp20_and_in_constexpr()) { + value128 answer; + answer.low = umul128_generic(a, b, &answer.high); + return answer; + } + value128 answer; +#if defined(_M_ARM64) && !defined(__MINGW32__) + // ARM64 has native support for 64-bit multiplications, no need to emulate + // But MinGW on ARM64 doesn't have native support for 64-bit multiplications + answer.high = __umulh(a, b); + answer.low = a * b; +#elif defined(SIMDJSON_FASTFLOAT_32BIT) || (defined(_WIN64) && !defined(__clang__) && \ + !defined(_M_ARM64) && !defined(__GNUC__)) + answer.low = _umul128(a, b, &answer.high); // _umul128 not available on ARM64 +#elif defined(SIMDJSON_FASTFLOAT_64BIT) && defined(__SIZEOF_INT128__) + __uint128_t r = static_cast<__uint128_t>(a) * b; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#else + answer.low = umul128_generic(a, b, &answer.high); +#endif + return answer; +} + +struct adjusted_mantissa { + uint64_t mantissa{0}; + int32_t power2{0}; // a negative value indicates an invalid result + adjusted_mantissa() = default; + + constexpr bool operator==(adjusted_mantissa const &o) const { + return mantissa == o.mantissa && power2 == o.power2; + } + + constexpr bool operator!=(adjusted_mantissa const &o) const { + return mantissa != o.mantissa || power2 != o.power2; + } +}; + +// Bias so we can get the real exponent with an invalid adjusted_mantissa. +constexpr static int32_t invalid_am_bias = -0x8000; + +// used for binary_format_lookup_tables::max_mantissa +constexpr uint64_t constant_55555 = 5 * 5 * 5 * 5 * 5; + +template struct binary_format_lookup_tables; + +template struct binary_format : binary_format_lookup_tables { + using equiv_uint = equiv_uint_t; + + static constexpr int mantissa_explicit_bits(); + static constexpr int minimum_exponent(); + static constexpr int infinite_power(); + static constexpr int sign_index(); + static constexpr int + min_exponent_fast_path(); // used when fegetround() == FE_TONEAREST + static constexpr int max_exponent_fast_path(); + static constexpr int max_exponent_round_to_even(); + static constexpr int min_exponent_round_to_even(); + static constexpr uint64_t max_mantissa_fast_path(int64_t power); + static constexpr uint64_t + max_mantissa_fast_path(); // used when fegetround() == FE_TONEAREST + static constexpr int largest_power_of_ten(); + static constexpr int smallest_power_of_ten(); + static constexpr T exact_power_of_ten(int64_t power); + static constexpr size_t max_digits(); + static constexpr equiv_uint exponent_mask(); + static constexpr equiv_uint mantissa_mask(); + static constexpr equiv_uint hidden_bit_mask(); +}; + +template struct binary_format_lookup_tables { + static constexpr double powers_of_ten[] = { + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22}; + + // Largest integer value v so that (5**index * v) <= 1<<53. + // 0x20000000000000 == 1 << 53 + static constexpr uint64_t max_mantissa[] = { + 0x20000000000000, + 0x20000000000000 / 5, + 0x20000000000000 / (5 * 5), + 0x20000000000000 / (5 * 5 * 5), + 0x20000000000000 / (5 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555), + 0x20000000000000 / (constant_55555 * 5), + 0x20000000000000 / (constant_55555 * 5 * 5), + 0x20000000000000 / (constant_55555 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555 * 5 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555), + 0x20000000000000 / (constant_55555 * constant_55555 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * 5), + 0x20000000000000 / + (constant_55555 * constant_55555 * constant_55555 * 5 * 5), + 0x20000000000000 / + (constant_55555 * constant_55555 * constant_55555 * 5 * 5 * 5), + 0x20000000000000 / + (constant_55555 * constant_55555 * constant_55555 * 5 * 5 * 5 * 5), + 0x20000000000000 / + (constant_55555 * constant_55555 * constant_55555 * constant_55555), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * + constant_55555 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * + constant_55555 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * + constant_55555 * 5 * 5 * 5), + 0x20000000000000 / (constant_55555 * constant_55555 * constant_55555 * + constant_55555 * 5 * 5 * 5 * 5)}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template +constexpr double binary_format_lookup_tables::powers_of_ten[]; + +template +constexpr uint64_t binary_format_lookup_tables::max_mantissa[]; + +#endif + +template struct binary_format_lookup_tables { + static constexpr float powers_of_ten[] = {1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + + // Largest integer value v so that (5**index * v) <= 1<<24. + // 0x1000000 == 1<<24 + static constexpr uint64_t max_mantissa[] = { + 0x1000000, + 0x1000000 / 5, + 0x1000000 / (5 * 5), + 0x1000000 / (5 * 5 * 5), + 0x1000000 / (5 * 5 * 5 * 5), + 0x1000000 / (constant_55555), + 0x1000000 / (constant_55555 * 5), + 0x1000000 / (constant_55555 * 5 * 5), + 0x1000000 / (constant_55555 * 5 * 5 * 5), + 0x1000000 / (constant_55555 * 5 * 5 * 5 * 5), + 0x1000000 / (constant_55555 * constant_55555), + 0x1000000 / (constant_55555 * constant_55555 * 5)}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template +constexpr float binary_format_lookup_tables::powers_of_ten[]; + +template +constexpr uint64_t binary_format_lookup_tables::max_mantissa[]; + +#endif + +template <> +inline constexpr int binary_format::min_exponent_fast_path() { +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + return 0; +#else + return -22; +#endif +} + +template <> +inline constexpr int binary_format::min_exponent_fast_path() { +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + return 0; +#else + return -10; +#endif +} + +template <> +inline constexpr int binary_format::mantissa_explicit_bits() { + return 52; +} + +template <> +inline constexpr int binary_format::mantissa_explicit_bits() { + return 23; +} + +template <> +inline constexpr int binary_format::max_exponent_round_to_even() { + return 23; +} + +template <> +inline constexpr int binary_format::max_exponent_round_to_even() { + return 10; +} + +template <> +inline constexpr int binary_format::min_exponent_round_to_even() { + return -4; +} + +template <> +inline constexpr int binary_format::min_exponent_round_to_even() { + return -17; +} + +template <> inline constexpr int binary_format::minimum_exponent() { + return -1023; +} + +template <> inline constexpr int binary_format::minimum_exponent() { + return -127; +} + +template <> inline constexpr int binary_format::infinite_power() { + return 0x7FF; +} + +template <> inline constexpr int binary_format::infinite_power() { + return 0xFF; +} + +template <> inline constexpr int binary_format::sign_index() { + return 63; +} + +template <> inline constexpr int binary_format::sign_index() { + return 31; +} + +template <> +inline constexpr int binary_format::max_exponent_fast_path() { + return 22; +} + +template <> +inline constexpr int binary_format::max_exponent_fast_path() { + return 10; +} + +template <> +inline constexpr uint64_t binary_format::max_mantissa_fast_path() { + return uint64_t(2) << mantissa_explicit_bits(); +} + +template <> +inline constexpr uint64_t binary_format::max_mantissa_fast_path() { + return uint64_t(2) << mantissa_explicit_bits(); +} + +// credit: Jakub Jelinek +#ifdef __STDCPP_FLOAT16_T__ +template struct binary_format_lookup_tables { + static constexpr std::float16_t powers_of_ten[] = {1e0f16, 1e1f16, 1e2f16, + 1e3f16, 1e4f16}; + + // Largest integer value v so that (5**index * v) <= 1<<11. + // 0x800 == 1<<11 + static constexpr uint64_t max_mantissa[] = {0x800, + 0x800 / 5, + 0x800 / (5 * 5), + 0x800 / (5 * 5 * 5), + 0x800 / (5 * 5 * 5 * 5), + 0x800 / (constant_55555)}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template +constexpr std::float16_t + binary_format_lookup_tables::powers_of_ten[]; + +template +constexpr uint64_t + binary_format_lookup_tables::max_mantissa[]; + +#endif + +template <> +inline constexpr std::float16_t +binary_format::exact_power_of_ten(int64_t power) { + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(powers_of_ten[0]), powers_of_ten[power]; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::exponent_mask() { + return 0x7C00; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::mantissa_mask() { + return 0x03FF; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::hidden_bit_mask() { + return 0x0400; +} + +template <> +inline constexpr int binary_format::max_exponent_fast_path() { + return 4; +} + +template <> +inline constexpr int binary_format::mantissa_explicit_bits() { + return 10; +} + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path() { + return uint64_t(2) << mantissa_explicit_bits(); +} + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path(int64_t power) { + // caller is responsible to ensure that + // power >= 0 && power <= 4 + // + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(max_mantissa[0]), max_mantissa[power]; +} + +template <> +inline constexpr int binary_format::min_exponent_fast_path() { + return 0; +} + +template <> +inline constexpr int +binary_format::max_exponent_round_to_even() { + return 5; +} + +template <> +inline constexpr int +binary_format::min_exponent_round_to_even() { + return -22; +} + +template <> +inline constexpr int binary_format::minimum_exponent() { + return -15; +} + +template <> +inline constexpr int binary_format::infinite_power() { + return 0x1F; +} + +template <> inline constexpr int binary_format::sign_index() { + return 15; +} + +template <> +inline constexpr int binary_format::largest_power_of_ten() { + return 4; +} + +template <> +inline constexpr int binary_format::smallest_power_of_ten() { + return -27; +} + +template <> +inline constexpr size_t binary_format::max_digits() { + return 22; +} +#endif // __STDCPP_FLOAT16_T__ + +// credit: Jakub Jelinek +#ifdef __STDCPP_BFLOAT16_T__ +template struct binary_format_lookup_tables { + static constexpr std::bfloat16_t powers_of_ten[] = {1e0bf16, 1e1bf16, 1e2bf16, + 1e3bf16}; + + // Largest integer value v so that (5**index * v) <= 1<<8. + // 0x100 == 1<<8 + static constexpr uint64_t max_mantissa[] = {0x100, 0x100 / 5, 0x100 / (5 * 5), + 0x100 / (5 * 5 * 5), + 0x100 / (5 * 5 * 5 * 5)}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template +constexpr std::bfloat16_t + binary_format_lookup_tables::powers_of_ten[]; + +template +constexpr uint64_t + binary_format_lookup_tables::max_mantissa[]; + +#endif + +template <> +inline constexpr std::bfloat16_t +binary_format::exact_power_of_ten(int64_t power) { + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(powers_of_ten[0]), powers_of_ten[power]; +} + +template <> +inline constexpr int binary_format::max_exponent_fast_path() { + return 3; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::exponent_mask() { + return 0x7F80; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::mantissa_mask() { + return 0x007F; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::hidden_bit_mask() { + return 0x0080; +} + +template <> +inline constexpr int binary_format::mantissa_explicit_bits() { + return 7; +} + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path() { + return uint64_t(2) << mantissa_explicit_bits(); +} + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path(int64_t power) { + // caller is responsible to ensure that + // power >= 0 && power <= 3 + // + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(max_mantissa[0]), max_mantissa[power]; +} + +template <> +inline constexpr int binary_format::min_exponent_fast_path() { + return 0; +} + +template <> +inline constexpr int +binary_format::max_exponent_round_to_even() { + return 3; +} + +template <> +inline constexpr int +binary_format::min_exponent_round_to_even() { + return -24; +} + +template <> +inline constexpr int binary_format::minimum_exponent() { + return -127; +} + +template <> +inline constexpr int binary_format::infinite_power() { + return 0xFF; +} + +template <> inline constexpr int binary_format::sign_index() { + return 15; +} + +template <> +inline constexpr int binary_format::largest_power_of_ten() { + return 38; +} + +template <> +inline constexpr int binary_format::smallest_power_of_ten() { + return -60; +} + +template <> +inline constexpr size_t binary_format::max_digits() { + return 98; +} +#endif // __STDCPP_BFLOAT16_T__ + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path(int64_t power) { + // caller is responsible to ensure that + // power >= 0 && power <= 22 + // + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(max_mantissa[0]), max_mantissa[power]; +} + +template <> +inline constexpr uint64_t +binary_format::max_mantissa_fast_path(int64_t power) { + // caller is responsible to ensure that + // power >= 0 && power <= 10 + // + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(max_mantissa[0]), max_mantissa[power]; +} + +template <> +inline constexpr double +binary_format::exact_power_of_ten(int64_t power) { + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(powers_of_ten[0]), powers_of_ten[power]; +} + +template <> +inline constexpr float binary_format::exact_power_of_ten(int64_t power) { + // Work around clang bug https://godbolt.org/z/zedh7rrhc + return static_cast(powers_of_ten[0]), powers_of_ten[power]; +} + +template <> inline constexpr int binary_format::largest_power_of_ten() { + return 308; +} + +template <> inline constexpr int binary_format::largest_power_of_ten() { + return 38; +} + +template <> +inline constexpr int binary_format::smallest_power_of_ten() { + return -342; +} + +template <> inline constexpr int binary_format::smallest_power_of_ten() { + return -64; +} + +template <> inline constexpr size_t binary_format::max_digits() { + return 769; +} + +template <> inline constexpr size_t binary_format::max_digits() { + return 114; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::exponent_mask() { + return 0x7F800000; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::exponent_mask() { + return 0x7FF0000000000000; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::mantissa_mask() { + return 0x007FFFFF; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::mantissa_mask() { + return 0x000FFFFFFFFFFFFF; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::hidden_bit_mask() { + return 0x00800000; +} + +template <> +inline constexpr binary_format::equiv_uint +binary_format::hidden_bit_mask() { + return 0x0010000000000000; +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +to_float(bool negative, adjusted_mantissa am, T &value) { + using equiv_uint = equiv_uint_t; + equiv_uint word = equiv_uint(am.mantissa); + word = equiv_uint(word | equiv_uint(am.power2) + << binary_format::mantissa_explicit_bits()); + word = + equiv_uint(word | equiv_uint(negative) << binary_format::sign_index()); +#if SIMDJSON_FASTFLOAT_HAS_BIT_CAST + value = std::bit_cast(word); +#else + ::memcpy(&value, &word, sizeof(T)); +#endif +} + +template struct space_lut { + static constexpr bool value[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template constexpr bool space_lut::value[]; + +#endif + +template constexpr bool is_space(UC c) { + // wchar_t and char can be signed, so a negative code unit slips past a plain + // `c < 256` and then indexes the table by its truncated low byte. Compare as + // unsigned, matching the care taken in ch_to_digit. + using UnsignedUC = typename std::make_unsigned::type; + return static_cast(c) < 256 && space_lut<>::value[uint8_t(c)]; +} + +template static constexpr uint64_t int_cmp_zeros() { + static_assert((sizeof(UC) == 1) || (sizeof(UC) == 2) || (sizeof(UC) == 4), + "Unsupported character size"); + return (sizeof(UC) == 1) ? 0x3030303030303030 + : (sizeof(UC) == 2) + ? (uint64_t(UC('0')) << 48 | uint64_t(UC('0')) << 32 | + uint64_t(UC('0')) << 16 | UC('0')) + : (uint64_t(UC('0')) << 32 | UC('0')); +} + +template static constexpr int int_cmp_len() { + return sizeof(uint64_t) / sizeof(UC); +} + +template constexpr UC const *str_const_nan(); + +template <> constexpr char const *str_const_nan() { return "nan"; } + +template <> constexpr wchar_t const *str_const_nan() { return L"nan"; } + +template <> constexpr char16_t const *str_const_nan() { + return u"nan"; +} + +template <> constexpr char32_t const *str_const_nan() { + return U"nan"; +} + +#ifdef __cpp_char8_t +template <> constexpr char8_t const *str_const_nan() { + return u8"nan"; +} +#endif + +template constexpr UC const *str_const_inf(); + +template <> constexpr char const *str_const_inf() { return "infinity"; } + +template <> constexpr wchar_t const *str_const_inf() { + return L"infinity"; +} + +template <> constexpr char16_t const *str_const_inf() { + return u"infinity"; +} + +template <> constexpr char32_t const *str_const_inf() { + return U"infinity"; +} + +#ifdef __cpp_char8_t +template <> constexpr char8_t const *str_const_inf() { + return u8"infinity"; +} +#endif + +template struct int_luts { + static constexpr uint8_t chdigit[] = { + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, + 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 255, 255, 255, 255, 255, 255, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, + 33, 34, 35, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255}; + + static constexpr size_t maxdigits_u64[] = { + 64, 41, 32, 28, 25, 23, 22, 21, 20, 19, 18, 18, 17, 17, 16, 16, 16, 16, + 15, 15, 15, 15, 14, 14, 14, 14, 14, 14, 14, 13, 13, 13, 13, 13, 13}; + + static constexpr uint64_t min_safe_u64[] = { + 9223372036854775808ull, 12157665459056928801ull, 4611686018427387904, + 7450580596923828125, 4738381338321616896, 3909821048582988049, + 9223372036854775808ull, 12157665459056928801ull, 10000000000000000000ull, + 5559917313492231481, 2218611106740436992, 8650415919381337933, + 2177953337809371136, 6568408355712890625, 1152921504606846976, + 2862423051509815793, 6746640616477458432, 15181127029874798299ull, + 1638400000000000000, 3243919932521508681, 6221821273427820544, + 11592836324538749809ull, 876488338465357824, 1490116119384765625, + 2481152873203736576, 4052555153018976267, 6502111422497947648, + 10260628712958602189ull, 15943230000000000000ull, 787662783788549761, + 1152921504606846976, 1667889514952984961, 2386420683693101056, + 3379220508056640625, 4738381338321616896}; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template constexpr uint8_t int_luts::chdigit[]; + +template constexpr size_t int_luts::maxdigits_u64[]; + +template constexpr uint64_t int_luts::min_safe_u64[]; + +#endif + +template +simdjson_fastfloat_really_inline constexpr uint8_t ch_to_digit(UC c) { + // wchar_t and char can be signed, so we need to be careful. + using UnsignedUC = typename std::make_unsigned::type; + return int_luts<>::chdigit[static_cast( + static_cast(c) & + static_cast( + -((static_cast(c) & ~0xFFull) == 0)))]; +} + +simdjson_fastfloat_really_inline constexpr size_t max_digits_u64(int base) { + return int_luts<>::maxdigits_u64[base - 2]; +} + +// If a u64 is exactly max_digits_u64() in length, this is +// the value below which it has definitely overflowed. +simdjson_fastfloat_really_inline constexpr uint64_t min_safe_u64(int base) { + return int_luts<>::min_safe_u64[base - 2]; +} + +static_assert(std::is_same, uint64_t>::value, + "equiv_uint should be uint64_t for double"); +static_assert(std::numeric_limits::is_iec559, + "double must fulfill the requirements of IEC 559 (IEEE 754)"); + +static_assert(std::is_same, uint32_t>::value, + "equiv_uint should be uint32_t for float"); +static_assert(std::numeric_limits::is_iec559, + "float must fulfill the requirements of IEC 559 (IEEE 754)"); + +#ifdef __STDCPP_FLOAT64_T__ +static_assert(std::is_same, uint64_t>::value, + "equiv_uint should be uint64_t for std::float64_t"); +static_assert( + std::numeric_limits::is_iec559, + "std::float64_t must fulfill the requirements of IEC 559 (IEEE 754)"); + +template <> +struct binary_format : public binary_format {}; +#endif // __STDCPP_FLOAT64_T__ + +#ifdef __STDCPP_FLOAT32_T__ +static_assert(std::is_same, uint32_t>::value, + "equiv_uint should be uint32_t for std::float32_t"); +static_assert( + std::numeric_limits::is_iec559, + "std::float32_t must fulfill the requirements of IEC 559 (IEEE 754)"); + +template <> +struct binary_format : public binary_format {}; +#endif // __STDCPP_FLOAT32_T__ + +#ifdef __STDCPP_FLOAT16_T__ +static_assert( + std::is_same::equiv_uint, uint16_t>::value, + "equiv_uint should be uint16_t for std::float16_t"); +static_assert( + std::numeric_limits::is_iec559, + "std::float16_t must fulfill the requirements of IEC 559 (IEEE 754)"); +#endif // __STDCPP_FLOAT16_T__ + +#ifdef __STDCPP_BFLOAT16_T__ +static_assert( + std::is_same::equiv_uint, uint16_t>::value, + "equiv_uint should be uint16_t for std::bfloat16_t"); +static_assert( + std::numeric_limits::is_iec559, + "std::bfloat16_t must fulfill the requirements of IEC 559 (IEEE 754)"); +#endif // __STDCPP_BFLOAT16_T__ + +constexpr chars_format operator~(chars_format rhs) noexcept { + using int_type = std::underlying_type::type; + return static_cast(~static_cast(rhs)); +} + +constexpr chars_format operator&(chars_format lhs, chars_format rhs) noexcept { + using int_type = std::underlying_type::type; + return static_cast(static_cast(lhs) & + static_cast(rhs)); +} + +constexpr chars_format operator|(chars_format lhs, chars_format rhs) noexcept { + using int_type = std::underlying_type::type; + return static_cast(static_cast(lhs) | + static_cast(rhs)); +} + +constexpr chars_format operator^(chars_format lhs, chars_format rhs) noexcept { + using int_type = std::underlying_type::type; + return static_cast(static_cast(lhs) ^ + static_cast(rhs)); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 chars_format & +operator&=(chars_format &lhs, chars_format rhs) noexcept { + return lhs = (lhs & rhs); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 chars_format & +operator|=(chars_format &lhs, chars_format rhs) noexcept { + return lhs = (lhs | rhs); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 chars_format & +operator^=(chars_format &lhs, chars_format rhs) noexcept { + return lhs = (lhs ^ rhs); +} + +namespace detail { +// adjust for deprecated feature macros +constexpr chars_format adjust_for_feature_macros(chars_format fmt) { + return fmt +#ifdef SIMDJSON_FASTFLOAT_ALLOWS_LEADING_PLUS + | chars_format::allow_leading_plus +#endif +#ifdef SIMDJSON_FASTFLOAT_SKIP_WHITE_SPACE + | chars_format::skip_white_space +#endif + ; +} +} // namespace detail +} // namespace simdjson_fast_float + +#endif + + +#ifndef SIMDJSON_FASTFLOAT_FAST_FLOAT_H +#define SIMDJSON_FASTFLOAT_FAST_FLOAT_H + + +namespace simdjson_fast_float { +/** + * This function parses the character sequence [first,last) for a number. It + * parses floating-point numbers expecting a locale-independent format + * equivalent to what is used by std::strtod in the default ("C") locale. The + * resulting floating-point value is the closest floating-point values (using + * either float or double), using the "round to even" convention for values that + * would otherwise fall right in-between two values. That is, we provide exact + * parsing according to the IEEE standard. + * + * Given a successful parse, the pointer (`ptr`) in the returned value is set to + * point right after the parsed number, and the `value` referenced is set to the + * parsed value. In case of error, the returned `ec` contains a representative + * error, otherwise the default (`std::errc()`) value is stored. + * + * The implementation does not throw and does not allocate memory (e.g., with + * `new` or `malloc`). + * + * Like the C++17 standard, the `simdjson_fast_float::from_chars` functions take an + * optional last argument of the type `simdjson_fast_float::chars_format`. It is a bitset + * value: we check whether `fmt & simdjson_fast_float::chars_format::fixed` and `fmt & + * simdjson_fast_float::chars_format::scientific` are set to determine whether we allow + * the fixed point and scientific notation respectively. The default is + * `simdjson_fast_float::chars_format::general` which allows both `fixed` and + * `scientific`. + */ +template ::value)> +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars(UC const *first, UC const *last, T &value, + chars_format fmt = chars_format::general) noexcept; + +/** + * Like from_chars, but accepts an `options` argument to govern number parsing. + * Both for floating-point types and integer types. + */ +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_advanced(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept; + +/** + * This function multiplies an integer number by a power of 10 and returns + * the result as a double precision floating-point value that is correctly + * rounded. The resulting floating-point value is the closest floating-point + * value, using the "round to nearest, tie to even" convention for values that + * would otherwise fall right in-between two values. That is, we provide exact + * conversion according to the IEEE standard. + * + * On overflow infinity is returned, on underflow 0 is returned. + * + * The implementation does not throw and does not allocate memory (e.g., with + * `new` or `malloc`). + */ +SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double +integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept; +SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double +integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept; + +/** + * This function is a template overload of `integer_times_pow10()` + * that returns a floating-point value of type `T` that is one of + * supported floating-point types (e.g. `double`, `float`). + */ +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value, T>::type + integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept; +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value, T>::type + integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept; + +/** + * from_chars for integer types. + */ +template ::value)> +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars(UC const *first, UC const *last, T &value, int base = 10) noexcept; + +} // namespace simdjson_fast_float + +#endif // SIMDJSON_FASTFLOAT_FAST_FLOAT_H + +#ifndef SIMDJSON_FASTFLOAT_ASCII_NUMBER_H +#define SIMDJSON_FASTFLOAT_ASCII_NUMBER_H + +#include +#include +#include +#include +#include +#include + + +#ifdef SIMDJSON_FASTFLOAT_SSE2 +#include +#endif + +#ifdef SIMDJSON_FASTFLOAT_NEON +#include +#endif + +namespace simdjson_fast_float { + +template simdjson_fastfloat_really_inline constexpr bool has_simd_opt() { +#ifdef SIMDJSON_FASTFLOAT_HAS_SIMD + return std::is_same::value; +#else + return false; +#endif +} + +// Next function can be micro-optimized, but compilers are entirely +// able to optimize it well. +template +simdjson_fastfloat_really_inline constexpr bool is_integer(UC c) noexcept { + return static_cast(c - UC('0')) <= 9u; +} + +simdjson_fastfloat_really_inline constexpr uint64_t byteswap(uint64_t val) { + return (val & 0xFF00000000000000) >> 56 | (val & 0x00FF000000000000) >> 40 | + (val & 0x0000FF0000000000) >> 24 | (val & 0x000000FF00000000) >> 8 | + (val & 0x00000000FF000000) << 8 | (val & 0x0000000000FF0000) << 24 | + (val & 0x000000000000FF00) << 40 | (val & 0x00000000000000FF) << 56; +} + +simdjson_fastfloat_really_inline constexpr uint32_t byteswap_32(uint32_t val) { + return (val >> 24) | ((val >> 8) & 0x0000FF00u) | ((val << 8) & 0x00FF0000u) | + (val << 24); +} + +// Read 8 UC into a u64. Truncates UC if not char. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +read8_to_u64(UC const *chars) { + if (cpp20_and_in_constexpr() || !std::is_same::value) { + uint64_t val = 0; + for (int i = 0; i < 8; ++i) { + val |= uint64_t(uint8_t(*chars)) << (i * 8); + ++chars; + } + return val; + } + uint64_t val; + ::memcpy(&val, chars, sizeof(uint64_t)); +#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN == 1 + // Need to read as-if the number was in little-endian order. + val = byteswap(val); +#endif + return val; +} + +// Read 4 UC into a u32. Truncates UC if not char. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint32_t +read4_to_u32(UC const *chars) { + if (cpp20_and_in_constexpr() || !std::is_same::value) { + uint32_t val = 0; + for (int i = 0; i < 4; ++i) { + val |= uint32_t(uint8_t(*chars)) << (i * 8); + ++chars; + } + return val; + } + uint32_t val; + ::memcpy(&val, chars, sizeof(uint32_t)); +#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN == 1 + val = byteswap_32(val); +#endif + return val; +} +#ifdef SIMDJSON_FASTFLOAT_SSE2 + +simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(__m128i const data) { + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + __m128i const packed = _mm_packus_epi16(data, data); +#ifdef SIMDJSON_FASTFLOAT_64BIT + return uint64_t(_mm_cvtsi128_si64(packed)); +#else + uint64_t value; + // Visual Studio + older versions of GCC don't support _mm_storeu_si64 + _mm_storel_epi64(reinterpret_cast<__m128i *>(&value), packed); + return value; +#endif + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +} + +simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) { + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + return simd_read8_to_u64( + _mm_loadu_si128(reinterpret_cast<__m128i const *>(chars))); + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +} + +#elif defined(SIMDJSON_FASTFLOAT_NEON) + +simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(uint16x8_t const data) { + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + uint8x8_t utf8_packed = vmovn_u16(data); + return vget_lane_u64(vreinterpret_u64_u8(utf8_packed), 0); + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +} + +simdjson_fastfloat_really_inline uint64_t simd_read8_to_u64(char16_t const *chars) { + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + return simd_read8_to_u64( + vld1q_u16(reinterpret_cast(chars))); + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +} + +#endif // SIMDJSON_FASTFLOAT_SSE2 + +// MSVC SFINAE is broken pre-VS2017 +#if defined(_MSC_VER) && _MSC_VER <= 1900 +template +#else +template ()) = 0> +#endif +// dummy for compile +uint64_t simd_read8_to_u64(UC const *) { + return 0; +} + +// credit @aqrit +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint32_t +parse_eight_digits_unrolled(uint64_t val) { + uint64_t const mask = 0x000000FF000000FF; + uint64_t const mul1 = 0x000F424000000064; // 100 + (1000000ULL << 32) + uint64_t const mul2 = 0x0000271000000001; // 1 + (10000ULL << 32) + val -= 0x3030303030303030; + val = (val * 10) + (val >> 8); // val = (val * 2561) >> 8; + val = (((val & mask) * mul1) + (((val >> 16) & mask) * mul2)) >> 32; + return uint32_t(val); +} + +// Call this if chars are definitely 8 digits. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint32_t +parse_eight_digits_unrolled(UC const *chars) noexcept { + if (cpp20_and_in_constexpr() || !has_simd_opt()) { + return parse_eight_digits_unrolled(read8_to_u64(chars)); // truncation okay + } + return parse_eight_digits_unrolled(simd_read8_to_u64(chars)); +} + +// credit @aqrit +simdjson_fastfloat_really_inline constexpr bool +is_made_of_eight_digits_fast(uint64_t val) noexcept { + return !((((val + 0x4646464646464646) | (val - 0x3030303030303030)) & + 0x8080808080808080)); +} + +simdjson_fastfloat_really_inline constexpr bool +is_made_of_four_digits_fast(uint32_t val) noexcept { + return !((((val + 0x46464646) | (val - 0x30303030)) & 0x80808080)); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint32_t +parse_four_digits_unrolled(uint32_t val) noexcept { + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + +#ifdef SIMDJSON_FASTFLOAT_HAS_SIMD + +// Call this if chars might not be 8 digits. +// Using this style (instead of is_made_of_eight_digits_fast() then +// parse_eight_digits_unrolled()) ensures we don't load SIMD registers twice. +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +simd_parse_if_eight_digits_unrolled(char16_t const *chars, + uint64_t &i) noexcept { + if (cpp20_and_in_constexpr()) { + return false; + } +#ifdef SIMDJSON_FASTFLOAT_SSE2 + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + __m128i const data = + _mm_loadu_si128(reinterpret_cast<__m128i const *>(chars)); + + // (x - '0') <= 9 + // http://0x80.pl/articles/simd-parsing-int-sequences.html + __m128i const t0 = _mm_add_epi16(data, _mm_set1_epi16(32720)); + __m128i const t1 = _mm_cmpgt_epi16(t0, _mm_set1_epi16(-32759)); + + if (_mm_movemask_epi8(t1) == 0) { + i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data)); + return true; + } else + return false; + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +#elif defined(SIMDJSON_FASTFLOAT_NEON) + SIMDJSON_FASTFLOAT_SIMD_DISABLE_WARNINGS + uint16x8_t const data = vld1q_u16(reinterpret_cast(chars)); + + // (x - '0') <= 9 + // http://0x80.pl/articles/simd-parsing-int-sequences.html + uint16x8_t const t0 = vsubq_u16(data, vmovq_n_u16('0')); + uint16x8_t const mask = vcltq_u16(t0, vmovq_n_u16('9' - '0' + 1)); + + if (vminvq_u16(mask) == 0xFFFF) { + i = i * 100000000 + parse_eight_digits_unrolled(simd_read8_to_u64(data)); + return true; + } else + return false; + SIMDJSON_FASTFLOAT_SIMD_RESTORE_WARNINGS +#else + static_cast(chars); + static_cast(i); + return false; +#endif // SIMDJSON_FASTFLOAT_SSE2 +} + +#endif // SIMDJSON_FASTFLOAT_HAS_SIMD + +// MSVC SFINAE is broken pre-VS2017 +#if defined(_MSC_VER) && _MSC_VER <= 1900 +template +#else +template ()) = 0> +#endif +// dummy for compile +bool simd_parse_if_eight_digits_unrolled(UC const *, uint64_t &) { + return 0; +} + +template ::value) = 0> +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +loop_parse_if_eight_digits(UC const *&p, UC const *const pend, uint64_t &i) { + if (!has_simd_opt()) { + return; + } + while ((std::distance(p, pend) >= 8) && + simd_parse_if_eight_digits_unrolled( + p, i)) { // in rare cases, this will overflow, but that's ok + p += 8; + } +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +loop_parse_if_eight_digits(char const *&p, char const *const pend, + uint64_t &i) { + // optimizes better than parse_if_eight_digits_unrolled() for UC = char. + while ((std::distance(p, pend) >= 8) && + is_made_of_eight_digits_fast(read8_to_u64(p))) { + i = i * 100000000 + + parse_eight_digits_unrolled(read8_to_u64( + p)); // in rare cases, this will overflow, but that's ok + p += 8; + } + // Consume a remaining 4-7 digit run in a single SWAR step instead of + // byte-by-byte (reuses the existing 4-digit helpers). The parsed result is + // identical either way. Historically gated to clang because gcc regressed on + // short remainders, but that verdict predates the span-elision restructure; + // with the leaner hot path the 4-digit step now wins on gcc as well. + if ((pend - p) >= 4) { + uint32_t const val4 = read4_to_u32(p); + if (is_made_of_four_digits_fast(val4)) { + i = i * 10000 + + parse_four_digits_unrolled(val4); // may overflow, that's ok + p += 4; + } + } +} + +enum class parse_error { + no_error, + // [JSON-only] The minus sign must be followed by an integer. + missing_integer_after_sign, + // A sign must be followed by an integer or dot. + missing_integer_or_dot_after_sign, + // [JSON-only] The integer part must not have leading zeros. + leading_zeros_in_integer_part, + // [JSON-only] The integer part must have at least one digit. + no_digits_in_integer_part, + // [JSON-only] If there is a decimal point, there must be digits in the + // fractional part. + no_digits_in_fractional_part, + // The mantissa must have at least one digit. + no_digits_in_mantissa, + // Scientific notation requires an exponential part. + missing_exponential_part, +}; + +template struct parsed_number_string_t { + int64_t exponent{0}; + uint64_t mantissa{0}; + UC const *lastmatch{nullptr}; + bool negative{false}; + bool valid{false}; + bool too_many_digits{false}; + // contains the range of the significant digits + span integer{}; // non-nullable + span fraction{}; // nullable + parse_error error{parse_error::no_error}; +}; + +using byte_span = span; +using parsed_number_string = parsed_number_string_t; + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 parsed_number_string_t +report_parse_error(UC const *p, parse_error error) { + parsed_number_string_t answer; + answer.valid = false; + answer.lastmatch = p; + answer.error = error; + return answer; +} + +// Assuming that you use no more than 19 digits, this will +// parse an ASCII string. +// +// store_spans is a *runtime* flag (not a template parameter, deliberately: a +// template would create a second instantiation of this whole function and the +// extra icache pressure wipes out the gain). When false, the integer/fraction +// spans (read only by the rare digit_comp slow path) are not materialized, +// which keeps the fat parsed_number_string_t off the hot path. The caller +// re-parses with store_spans=true if the slow path is actually reached. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 parsed_number_string_t +parse_number_string(UC const *p, UC const *pend, parse_options_t options, + bool store_spans = true) noexcept { + chars_format const fmt = detail::adjust_for_feature_macros(options.format); + UC const decimal_point = options.decimal_point; + + parsed_number_string_t answer; + answer.valid = false; + answer.too_many_digits = false; + // assume p < pend, so dereference without checks; + answer.negative = (*p == UC('-')); + // C++17 20.19.3.(7.1) explicitly forbids '+' sign here + if ((*p == UC('-')) || (uint64_t(fmt & chars_format::allow_leading_plus) && + !basic_json_fmt && *p == UC('+'))) { + ++p; + if (p == pend) { + return report_parse_error( + p, parse_error::missing_integer_or_dot_after_sign); + } + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(basic_json_fmt) { + if (!is_integer(*p)) { // a sign must be followed by an integer + return report_parse_error(p, + parse_error::missing_integer_after_sign); + } + } + else { + if (!is_integer(*p) && + (*p != + decimal_point)) { // a sign must be followed by an integer or the dot + return report_parse_error( + p, parse_error::missing_integer_or_dot_after_sign); + } + } + } + UC const *const start_digits = p; + + uint64_t i = 0; // an unsigned int avoids signed overflows (which are bad) + + // Straight-line unroll of the integer-part scan: most integer parts are + // 1-5 digits, so peeling the first iterations eliminates the loop back-edge + // for the common case. Semantics are identical to the original `while` loop: + // i = 10*i + digit, advancing p. + if ((p != pend) && is_integer(*p)) { + i = uint64_t(*p - UC('0')); + ++p; + if ((p != pend) && is_integer(*p)) { + i = 10 * i + uint64_t(*p - UC('0')); + ++p; + if ((p != pend) && is_integer(*p)) { + i = 10 * i + uint64_t(*p - UC('0')); + ++p; + if ((p != pend) && is_integer(*p)) { + i = 10 * i + uint64_t(*p - UC('0')); + ++p; + if ((p != pend) && is_integer(*p)) { + i = 10 * i + uint64_t(*p - UC('0')); + ++p; + while ((p != pend) && is_integer(*p)) { + // a multiplication by 10 is cheaper than an arbitrary integer + // multiplication + i = 10 * i + + uint64_t(*p - UC('0')); // might overflow, handled later + ++p; + } + } + } + } + } + } + UC const *const end_of_integer_part = p; + int64_t digit_count = int64_t(end_of_integer_part - start_digits); + if (store_spans) { + answer.integer = span(start_digits, size_t(digit_count)); + } + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(basic_json_fmt) { + // at least 1 digit in integer part, without leading zeros + if (digit_count == 0) { + return report_parse_error(p, parse_error::no_digits_in_integer_part); + } + if ((start_digits[0] == UC('0') && digit_count > 1)) { + return report_parse_error(start_digits, + parse_error::leading_zeros_in_integer_part); + } + } + + int64_t exponent = 0; + bool const has_decimal_point = (p != pend) && (*p == decimal_point); + if (has_decimal_point) { + ++p; + UC const *before = p; + // can occur at most twice without overflowing, but let it occur more, since + // for integers with many digits, digit parsing is the primary bottleneck. + loop_parse_if_eight_digits(p, pend, i); + + while ((p != pend) && is_integer(*p)) { + uint8_t digit = uint8_t(*p - UC('0')); + ++p; + i = i * 10 + digit; // in rare cases, this will overflow, but that's ok + } + exponent = before - p; + if (store_spans) { + answer.fraction = span(before, size_t(p - before)); + } + digit_count -= exponent; + } + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17(basic_json_fmt) { + // at least 1 digit in fractional part + if (has_decimal_point && exponent == 0) { + return report_parse_error(p, + parse_error::no_digits_in_fractional_part); + } + } + else if (digit_count == 0) { // we must have encountered at least one integer! + return report_parse_error(p, parse_error::no_digits_in_mantissa); + } + int64_t exp_number = 0; // explicit exponential part + if ((uint64_t(fmt & chars_format::scientific) && (p != pend) && + ((UC('e') == *p) || (UC('E') == *p))) || + (uint64_t(fmt & detail::basic_fortran_fmt) && (p != pend) && + ((UC('+') == *p) || (UC('-') == *p) || (UC('d') == *p) || + (UC('D') == *p)))) { + UC const *location_of_e = p; + if ((UC('e') == *p) || (UC('E') == *p) || (UC('d') == *p) || + (UC('D') == *p)) { + ++p; + } + bool neg_exp = false; + if ((p != pend) && (UC('-') == *p)) { + neg_exp = true; + ++p; + } else if ((p != pend) && + (UC('+') == + *p)) { // '+' on exponent is allowed by C++17 20.19.3.(7.1) + ++p; + } + if ((p == pend) || !is_integer(*p)) { + if (!uint64_t(fmt & chars_format::fixed)) { + // The exponential part is invalid for scientific notation, so it must + // be a trailing token for fixed notation. However, fixed notation is + // disabled, so report a scientific notation error. + return report_parse_error(p, parse_error::missing_exponential_part); + } + // Otherwise, we will be ignoring the 'e'. + p = location_of_e; + } else { + while ((p != pend) && is_integer(*p)) { + uint8_t digit = uint8_t(*p - UC('0')); + if (exp_number < 0x10000000) { + exp_number = 10 * exp_number + digit; + } + ++p; + } + if (neg_exp) { + exp_number = -exp_number; + } + exponent += exp_number; + } + } else { + // If it scientific and not fixed, we have to bail out. + if (uint64_t(fmt & chars_format::scientific) && + !uint64_t(fmt & chars_format::fixed)) { + return report_parse_error(p, parse_error::missing_exponential_part); + } + } + answer.lastmatch = p; + answer.valid = true; + + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon. + // + // We can deal with up to 19 digits. + if (digit_count > 19) { // this is uncommon + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + // We need to be mindful of the case where we only have zeroes... + // E.g., 0.000000000...000. + UC const *start = start_digits; + while ((start != pend) && (*start == UC('0') || *start == decimal_point)) { + if (*start == UC('0')) { + digit_count--; + } + start++; + } + + if (digit_count > 19) { + answer.too_many_digits = true; + // The truncation recompute below reads the integer/fraction spans. When + // store_spans is false we didn't materialize them, so just flag + // too_many_digits; the caller re-parses with store_spans=true to obtain + // the corrected mantissa/exponent before taking the slow path. + if (store_spans) { + // Let us start again, this time, avoiding overflows. + // We don't need to call if is_integer, since we use the + // pre-tokenized spans from above. + i = 0; + p = answer.integer.ptr; + UC const *int_end = p + answer.integer.len(); + uint64_t const minimal_nineteen_digit_integer{1000000000000000000}; + while ((i < minimal_nineteen_digit_integer) && (p != int_end)) { + i = i * 10 + uint64_t(*p - UC('0')); + ++p; + } + if (i >= minimal_nineteen_digit_integer) { // We have a big integer + exponent = end_of_integer_part - p + exp_number; + } else { // We have a value with a fractional component. + p = answer.fraction.ptr; + UC const *frac_end = p + answer.fraction.len(); + while ((i < minimal_nineteen_digit_integer) && (p != frac_end)) { + i = i * 10 + uint64_t(*p - UC('0')); + ++p; + } + exponent = answer.fraction.ptr - p + exp_number; + } + // We have now corrected both exponent and i, to a truncated value + } + } + } + answer.exponent = exponent; + answer.mantissa = i; + return answer; +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +parse_int_string(UC const *p, UC const *pend, T &value, + parse_options_t options) { + chars_format const fmt = detail::adjust_for_feature_macros(options.format); + int const base = options.base; + + from_chars_result_t answer; + + UC const *const first = p; + + bool const negative = (*p == UC('-')); +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(push) +#pragma warning(disable : 4127) +#endif + if (!std::is_signed::value && negative) { +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(pop) +#endif + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + return answer; + } + if ((*p == UC('-')) || + (uint64_t(fmt & chars_format::allow_leading_plus) && (*p == UC('+')))) { + ++p; + } + + UC const *const start_num = p; + + while (p != pend && *p == UC('0')) { + ++p; + } + + bool const has_leading_zeros = p > start_num; + + UC const *const start_digits = p; + + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17( + (std::is_same::value && sizeof(UC) == 1)) { + if (base == 10) { + const size_t len = static_cast(pend - p); + if (len == 0) { + if (has_leading_zeros) { + value = 0; + answer.ec = std::errc(); + answer.ptr = p; + } else { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + } + return answer; + } + + uint32_t digits; + +#if SIMDJSON_FASTFLOAT_HAS_IS_CONSTANT_EVALUATED && SIMDJSON_FASTFLOAT_HAS_BIT_CAST + if (std::is_constant_evaluated()) { + uint8_t str[4]{}; + for (size_t j = 0; j < 4 && j < len; ++j) { + str[j] = static_cast(p[j]); + } + digits = std::bit_cast(str); +#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN + digits = byteswap_32(digits); +#endif + } +#else + if (false) { + } +#endif + else if (len >= 4) { + ::memcpy(&digits, p, 4); +#if SIMDJSON_FASTFLOAT_IS_BIG_ENDIAN + digits = byteswap_32(digits); +#endif + } else { + uint32_t b0 = static_cast(p[0]); + uint32_t b1 = (len > 1) ? static_cast(p[1]) : 0xFFu; + uint32_t b2 = (len > 2) ? static_cast(p[2]) : 0xFFu; + uint32_t b3 = 0xFFu; + digits = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24); + } + + uint32_t magic = + ((digits + 0x46464646u) | (digits - 0x30303030u)) & 0x80808080u; + uint32_t tz = + static_cast(countr_zero_32(magic)); // 7, 15, 23, 31, or 32 + uint32_t nd = (tz == 32) ? 4 : (tz >> 3); + nd = static_cast(nd < len ? nd : len); + if (nd == 0) { + if (has_leading_zeros) { + value = 0; + answer.ec = std::errc(); + answer.ptr = p; + return answer; + } + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + return answer; + } + if (nd > 3) { + const UC *q = p + nd; + size_t rem = len - nd; + while (rem) { + if (*q < UC('0') || *q > UC('9')) + break; + ++q; + --rem; + } + answer.ec = std::errc::result_out_of_range; + answer.ptr = q; + return answer; + } + + digits ^= 0x30303030u; + digits <<= ((4 - nd) * 8); + + uint32_t check = ((digits >> 24) & 0xff) | ((digits >> 8) & 0xff00) | + ((digits << 8) & 0xff0000); + if (check > 0x00020505) { + answer.ec = std::errc::result_out_of_range; + answer.ptr = p + nd; + return answer; + } + value = static_cast((0x640a01 * digits) >> 24); + answer.ec = std::errc(); + answer.ptr = p + nd; + return answer; + } + } + + SIMDJSON_FASTFLOAT_IF_CONSTEXPR17( + (std::is_same::value && sizeof(UC) == 1)) { + if (base == 10) { + const size_t len = size_t(pend - p); + if (len == 0) { + if (has_leading_zeros) { + value = 0; + answer.ec = std::errc(); + answer.ptr = p; + } else { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + } + return answer; + } + + if (len >= 4) { + uint32_t digits = read4_to_u32(p); + if (is_made_of_four_digits_fast(digits)) { + uint32_t v = parse_four_digits_unrolled(digits); + if (len >= 5 && is_integer(p[4])) { + v = v * 10 + uint32_t(p[4] - '0'); + if (len >= 6 && is_integer(p[5])) { + answer.ec = std::errc::result_out_of_range; + const UC *q = p + 5; + while (q != pend && is_integer(*q)) { + q++; + } + answer.ptr = q; + return answer; + } + if (v > 65535) { + answer.ec = std::errc::result_out_of_range; + answer.ptr = p + 5; + return answer; + } + value = uint16_t(v); + answer.ec = std::errc(); + answer.ptr = p + 5; + return answer; + } + // 4 digits + value = uint16_t(v); + answer.ec = std::errc(); + answer.ptr = p + 4; + return answer; + } + } + } + } + + uint64_t i = 0; + if (base == 10) { + loop_parse_if_eight_digits(p, pend, i); // use SIMD if possible + } + while (p != pend) { + uint8_t digit = ch_to_digit(*p); + if (digit >= base) { + break; + } + i = uint64_t(base) * i + digit; // might overflow, check this later + p++; + } + + size_t digit_count = size_t(p - start_digits); + + if (digit_count == 0) { + if (has_leading_zeros) { + value = 0; + answer.ec = std::errc(); + answer.ptr = p; + } else { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + } + return answer; + } + + answer.ptr = p; + + // check u64 overflow + size_t max_digits = max_digits_u64(base); + if (digit_count > max_digits) { + answer.ec = std::errc::result_out_of_range; + return answer; + } + // this check can be eliminated for all other types, but they will all require + // a max_digits(base) equivalent + if (digit_count == max_digits) { + // At the max_digits boundary the accumulator `i` may have wrapped around + // 2^64. A plain `i < min_safe_u64(base)` test is not sufficient: for any + // base whose max_digits-length range exceeds 2^64 (base 10 reaches + // ~5.4 * 2^64 at 20 digits) the value can wrap a whole multiple of 2^64 and + // land back above min_safe, slipping through. Decide exactly in O(1) using + // the leading digit, following the approach used in simdjson: + // ms == min_safe_u64(base) == base^(max_digits-1), the smallest + // max_digits-length value. + // dmax == the largest leading digit whose number can still fit in u64. + // The leading-digit band [d*ms, (d+1)*ms) has width ms < 2^64, so within + // the single band where d == dmax the value straddles 2^64 at most once, + // and a single threshold separates wrapped from non-wrapped values. A + // leading digit above dmax always overflows; below dmax always fits. + uint64_t const ms = min_safe_u64(base); + uint64_t const dmax = (std::numeric_limits::max)() / ms; + uint64_t const lead = ch_to_digit(*start_digits); + if (lead > dmax || (lead == dmax && i < dmax * ms)) { + answer.ec = std::errc::result_out_of_range; + return answer; + } + } + + // check other types overflow + if (!std::is_same::value) { + if (i > uint64_t((std::numeric_limits::max)()) + uint64_t(negative)) { + answer.ec = std::errc::result_out_of_range; + return answer; + } + } + + if (negative) { +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(push) +#pragma warning(disable : 4146) +#endif + // this weird workaround is required because: + // - converting unsigned to signed when its value is greater than signed max + // is UB pre-C++23. + // - reinterpret_casting (~i + 1) would work, but it is not constexpr + // this is always optimized into a neg instruction (note: T is an integer + // type) + value = T(-(std::numeric_limits::max)() - + T(i - uint64_t((std::numeric_limits::max)()))); +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(pop) +#endif + } else { + value = T(i); + } + + answer.ec = std::errc(); + return answer; +} + +} // namespace simdjson_fast_float + +#endif + +#ifndef SIMDJSON_FASTFLOAT_FAST_TABLE_H +#define SIMDJSON_FASTFLOAT_FAST_TABLE_H + +#include + +namespace simdjson_fast_float { + +/** + * When mapping numbers from decimal to binary, + * we go from w * 10^q to m * 2^p but we have + * 10^q = 5^q * 2^q, so effectively + * we are trying to match + * w * 2^q * 5^q to m * 2^p. Thus the powers of two + * are not a concern since they can be represented + * exactly using the binary notation, only the powers of five + * affect the binary significand. + */ + +/** + * The smallest non-zero float (binary64) is 2^-1074. + * We take as input numbers of the form w x 10^q where w < 2^64. + * We have that w * 10^-343 < 2^(64-344) 5^-343 < 2^-1076. + * However, we have that + * (2^64-1) * 10^-342 = (2^64-1) * 2^-342 * 5^-342 > 2^-1074. + * Thus it is possible for a number of the form w * 10^-342 where + * w is a 64-bit value to be a non-zero floating-point number. + ********* + * Any number of form w * 10^309 where w>= 1 is going to be + * infinite in binary64 so we never need to worry about powers + * of 5 greater than 308. + */ +template struct powers_template { + + constexpr static int smallest_power_of_five = + binary_format::smallest_power_of_ten(); + constexpr static int largest_power_of_five = + binary_format::largest_power_of_ten(); + constexpr static int number_of_entries = + 2 * (largest_power_of_five - smallest_power_of_five + 1); + // Powers of five from 5^-342 all the way to 5^308 rounded toward one. + constexpr static uint64_t power_of_five_128[number_of_entries] = { + 0xeef453d6923bd65a, 0x113faa2906a13b3f, + 0x9558b4661b6565f8, 0x4ac7ca59a424c507, + 0xbaaee17fa23ebf76, 0x5d79bcf00d2df649, + 0xe95a99df8ace6f53, 0xf4d82c2c107973dc, + 0x91d8a02bb6c10594, 0x79071b9b8a4be869, + 0xb64ec836a47146f9, 0x9748e2826cdee284, + 0xe3e27a444d8d98b7, 0xfd1b1b2308169b25, + 0x8e6d8c6ab0787f72, 0xfe30f0f5e50e20f7, + 0xb208ef855c969f4f, 0xbdbd2d335e51a935, + 0xde8b2b66b3bc4723, 0xad2c788035e61382, + 0x8b16fb203055ac76, 0x4c3bcb5021afcc31, + 0xaddcb9e83c6b1793, 0xdf4abe242a1bbf3d, + 0xd953e8624b85dd78, 0xd71d6dad34a2af0d, + 0x87d4713d6f33aa6b, 0x8672648c40e5ad68, + 0xa9c98d8ccb009506, 0x680efdaf511f18c2, + 0xd43bf0effdc0ba48, 0x212bd1b2566def2, + 0x84a57695fe98746d, 0x14bb630f7604b57, + 0xa5ced43b7e3e9188, 0x419ea3bd35385e2d, + 0xcf42894a5dce35ea, 0x52064cac828675b9, + 0x818995ce7aa0e1b2, 0x7343efebd1940993, + 0xa1ebfb4219491a1f, 0x1014ebe6c5f90bf8, + 0xca66fa129f9b60a6, 0xd41a26e077774ef6, + 0xfd00b897478238d0, 0x8920b098955522b4, + 0x9e20735e8cb16382, 0x55b46e5f5d5535b0, + 0xc5a890362fddbc62, 0xeb2189f734aa831d, + 0xf712b443bbd52b7b, 0xa5e9ec7501d523e4, + 0x9a6bb0aa55653b2d, 0x47b233c92125366e, + 0xc1069cd4eabe89f8, 0x999ec0bb696e840a, + 0xf148440a256e2c76, 0xc00670ea43ca250d, + 0x96cd2a865764dbca, 0x380406926a5e5728, + 0xbc807527ed3e12bc, 0xc605083704f5ecf2, + 0xeba09271e88d976b, 0xf7864a44c633682e, + 0x93445b8731587ea3, 0x7ab3ee6afbe0211d, + 0xb8157268fdae9e4c, 0x5960ea05bad82964, + 0xe61acf033d1a45df, 0x6fb92487298e33bd, + 0x8fd0c16206306bab, 0xa5d3b6d479f8e056, + 0xb3c4f1ba87bc8696, 0x8f48a4899877186c, + 0xe0b62e2929aba83c, 0x331acdabfe94de87, + 0x8c71dcd9ba0b4925, 0x9ff0c08b7f1d0b14, + 0xaf8e5410288e1b6f, 0x7ecf0ae5ee44dd9, + 0xdb71e91432b1a24a, 0xc9e82cd9f69d6150, + 0x892731ac9faf056e, 0xbe311c083a225cd2, + 0xab70fe17c79ac6ca, 0x6dbd630a48aaf406, + 0xd64d3d9db981787d, 0x92cbbccdad5b108, + 0x85f0468293f0eb4e, 0x25bbf56008c58ea5, + 0xa76c582338ed2621, 0xaf2af2b80af6f24e, + 0xd1476e2c07286faa, 0x1af5af660db4aee1, + 0x82cca4db847945ca, 0x50d98d9fc890ed4d, + 0xa37fce126597973c, 0xe50ff107bab528a0, + 0xcc5fc196fefd7d0c, 0x1e53ed49a96272c8, + 0xff77b1fcbebcdc4f, 0x25e8e89c13bb0f7a, + 0x9faacf3df73609b1, 0x77b191618c54e9ac, + 0xc795830d75038c1d, 0xd59df5b9ef6a2417, + 0xf97ae3d0d2446f25, 0x4b0573286b44ad1d, + 0x9becce62836ac577, 0x4ee367f9430aec32, + 0xc2e801fb244576d5, 0x229c41f793cda73f, + 0xf3a20279ed56d48a, 0x6b43527578c1110f, + 0x9845418c345644d6, 0x830a13896b78aaa9, + 0xbe5691ef416bd60c, 0x23cc986bc656d553, + 0xedec366b11c6cb8f, 0x2cbfbe86b7ec8aa8, + 0x94b3a202eb1c3f39, 0x7bf7d71432f3d6a9, + 0xb9e08a83a5e34f07, 0xdaf5ccd93fb0cc53, + 0xe858ad248f5c22c9, 0xd1b3400f8f9cff68, + 0x91376c36d99995be, 0x23100809b9c21fa1, + 0xb58547448ffffb2d, 0xabd40a0c2832a78a, + 0xe2e69915b3fff9f9, 0x16c90c8f323f516c, + 0x8dd01fad907ffc3b, 0xae3da7d97f6792e3, + 0xb1442798f49ffb4a, 0x99cd11cfdf41779c, + 0xdd95317f31c7fa1d, 0x40405643d711d583, + 0x8a7d3eef7f1cfc52, 0x482835ea666b2572, + 0xad1c8eab5ee43b66, 0xda3243650005eecf, + 0xd863b256369d4a40, 0x90bed43e40076a82, + 0x873e4f75e2224e68, 0x5a7744a6e804a291, + 0xa90de3535aaae202, 0x711515d0a205cb36, + 0xd3515c2831559a83, 0xd5a5b44ca873e03, + 0x8412d9991ed58091, 0xe858790afe9486c2, + 0xa5178fff668ae0b6, 0x626e974dbe39a872, + 0xce5d73ff402d98e3, 0xfb0a3d212dc8128f, + 0x80fa687f881c7f8e, 0x7ce66634bc9d0b99, + 0xa139029f6a239f72, 0x1c1fffc1ebc44e80, + 0xc987434744ac874e, 0xa327ffb266b56220, + 0xfbe9141915d7a922, 0x4bf1ff9f0062baa8, + 0x9d71ac8fada6c9b5, 0x6f773fc3603db4a9, + 0xc4ce17b399107c22, 0xcb550fb4384d21d3, + 0xf6019da07f549b2b, 0x7e2a53a146606a48, + 0x99c102844f94e0fb, 0x2eda7444cbfc426d, + 0xc0314325637a1939, 0xfa911155fefb5308, + 0xf03d93eebc589f88, 0x793555ab7eba27ca, + 0x96267c7535b763b5, 0x4bc1558b2f3458de, + 0xbbb01b9283253ca2, 0x9eb1aaedfb016f16, + 0xea9c227723ee8bcb, 0x465e15a979c1cadc, + 0x92a1958a7675175f, 0xbfacd89ec191ec9, + 0xb749faed14125d36, 0xcef980ec671f667b, + 0xe51c79a85916f484, 0x82b7e12780e7401a, + 0x8f31cc0937ae58d2, 0xd1b2ecb8b0908810, + 0xb2fe3f0b8599ef07, 0x861fa7e6dcb4aa15, + 0xdfbdcece67006ac9, 0x67a791e093e1d49a, + 0x8bd6a141006042bd, 0xe0c8bb2c5c6d24e0, + 0xaecc49914078536d, 0x58fae9f773886e18, + 0xda7f5bf590966848, 0xaf39a475506a899e, + 0x888f99797a5e012d, 0x6d8406c952429603, + 0xaab37fd7d8f58178, 0xc8e5087ba6d33b83, + 0xd5605fcdcf32e1d6, 0xfb1e4a9a90880a64, + 0x855c3be0a17fcd26, 0x5cf2eea09a55067f, + 0xa6b34ad8c9dfc06f, 0xf42faa48c0ea481e, + 0xd0601d8efc57b08b, 0xf13b94daf124da26, + 0x823c12795db6ce57, 0x76c53d08d6b70858, + 0xa2cb1717b52481ed, 0x54768c4b0c64ca6e, + 0xcb7ddcdda26da268, 0xa9942f5dcf7dfd09, + 0xfe5d54150b090b02, 0xd3f93b35435d7c4c, + 0x9efa548d26e5a6e1, 0xc47bc5014a1a6daf, + 0xc6b8e9b0709f109a, 0x359ab6419ca1091b, + 0xf867241c8cc6d4c0, 0xc30163d203c94b62, + 0x9b407691d7fc44f8, 0x79e0de63425dcf1d, + 0xc21094364dfb5636, 0x985915fc12f542e4, + 0xf294b943e17a2bc4, 0x3e6f5b7b17b2939d, + 0x979cf3ca6cec5b5a, 0xa705992ceecf9c42, + 0xbd8430bd08277231, 0x50c6ff782a838353, + 0xece53cec4a314ebd, 0xa4f8bf5635246428, + 0x940f4613ae5ed136, 0x871b7795e136be99, + 0xb913179899f68584, 0x28e2557b59846e3f, + 0xe757dd7ec07426e5, 0x331aeada2fe589cf, + 0x9096ea6f3848984f, 0x3ff0d2c85def7621, + 0xb4bca50b065abe63, 0xfed077a756b53a9, + 0xe1ebce4dc7f16dfb, 0xd3e8495912c62894, + 0x8d3360f09cf6e4bd, 0x64712dd7abbbd95c, + 0xb080392cc4349dec, 0xbd8d794d96aacfb3, + 0xdca04777f541c567, 0xecf0d7a0fc5583a0, + 0x89e42caaf9491b60, 0xf41686c49db57244, + 0xac5d37d5b79b6239, 0x311c2875c522ced5, + 0xd77485cb25823ac7, 0x7d633293366b828b, + 0x86a8d39ef77164bc, 0xae5dff9c02033197, + 0xa8530886b54dbdeb, 0xd9f57f830283fdfc, + 0xd267caa862a12d66, 0xd072df63c324fd7b, + 0x8380dea93da4bc60, 0x4247cb9e59f71e6d, + 0xa46116538d0deb78, 0x52d9be85f074e608, + 0xcd795be870516656, 0x67902e276c921f8b, + 0x806bd9714632dff6, 0xba1cd8a3db53b6, + 0xa086cfcd97bf97f3, 0x80e8a40eccd228a4, + 0xc8a883c0fdaf7df0, 0x6122cd128006b2cd, + 0xfad2a4b13d1b5d6c, 0x796b805720085f81, + 0x9cc3a6eec6311a63, 0xcbe3303674053bb0, + 0xc3f490aa77bd60fc, 0xbedbfc4411068a9c, + 0xf4f1b4d515acb93b, 0xee92fb5515482d44, + 0x991711052d8bf3c5, 0x751bdd152d4d1c4a, + 0xbf5cd54678eef0b6, 0xd262d45a78a0635d, + 0xef340a98172aace4, 0x86fb897116c87c34, + 0x9580869f0e7aac0e, 0xd45d35e6ae3d4da0, + 0xbae0a846d2195712, 0x8974836059cca109, + 0xe998d258869facd7, 0x2bd1a438703fc94b, + 0x91ff83775423cc06, 0x7b6306a34627ddcf, + 0xb67f6455292cbf08, 0x1a3bc84c17b1d542, + 0xe41f3d6a7377eeca, 0x20caba5f1d9e4a93, + 0x8e938662882af53e, 0x547eb47b7282ee9c, + 0xb23867fb2a35b28d, 0xe99e619a4f23aa43, + 0xdec681f9f4c31f31, 0x6405fa00e2ec94d4, + 0x8b3c113c38f9f37e, 0xde83bc408dd3dd04, + 0xae0b158b4738705e, 0x9624ab50b148d445, + 0xd98ddaee19068c76, 0x3badd624dd9b0957, + 0x87f8a8d4cfa417c9, 0xe54ca5d70a80e5d6, + 0xa9f6d30a038d1dbc, 0x5e9fcf4ccd211f4c, + 0xd47487cc8470652b, 0x7647c3200069671f, + 0x84c8d4dfd2c63f3b, 0x29ecd9f40041e073, + 0xa5fb0a17c777cf09, 0xf468107100525890, + 0xcf79cc9db955c2cc, 0x7182148d4066eeb4, + 0x81ac1fe293d599bf, 0xc6f14cd848405530, + 0xa21727db38cb002f, 0xb8ada00e5a506a7c, + 0xca9cf1d206fdc03b, 0xa6d90811f0e4851c, + 0xfd442e4688bd304a, 0x908f4a166d1da663, + 0x9e4a9cec15763e2e, 0x9a598e4e043287fe, + 0xc5dd44271ad3cdba, 0x40eff1e1853f29fd, + 0xf7549530e188c128, 0xd12bee59e68ef47c, + 0x9a94dd3e8cf578b9, 0x82bb74f8301958ce, + 0xc13a148e3032d6e7, 0xe36a52363c1faf01, + 0xf18899b1bc3f8ca1, 0xdc44e6c3cb279ac1, + 0x96f5600f15a7b7e5, 0x29ab103a5ef8c0b9, + 0xbcb2b812db11a5de, 0x7415d448f6b6f0e7, + 0xebdf661791d60f56, 0x111b495b3464ad21, + 0x936b9fcebb25c995, 0xcab10dd900beec34, + 0xb84687c269ef3bfb, 0x3d5d514f40eea742, + 0xe65829b3046b0afa, 0xcb4a5a3112a5112, + 0x8ff71a0fe2c2e6dc, 0x47f0e785eaba72ab, + 0xb3f4e093db73a093, 0x59ed216765690f56, + 0xe0f218b8d25088b8, 0x306869c13ec3532c, + 0x8c974f7383725573, 0x1e414218c73a13fb, + 0xafbd2350644eeacf, 0xe5d1929ef90898fa, + 0xdbac6c247d62a583, 0xdf45f746b74abf39, + 0x894bc396ce5da772, 0x6b8bba8c328eb783, + 0xab9eb47c81f5114f, 0x66ea92f3f326564, + 0xd686619ba27255a2, 0xc80a537b0efefebd, + 0x8613fd0145877585, 0xbd06742ce95f5f36, + 0xa798fc4196e952e7, 0x2c48113823b73704, + 0xd17f3b51fca3a7a0, 0xf75a15862ca504c5, + 0x82ef85133de648c4, 0x9a984d73dbe722fb, + 0xa3ab66580d5fdaf5, 0xc13e60d0d2e0ebba, + 0xcc963fee10b7d1b3, 0x318df905079926a8, + 0xffbbcfe994e5c61f, 0xfdf17746497f7052, + 0x9fd561f1fd0f9bd3, 0xfeb6ea8bedefa633, + 0xc7caba6e7c5382c8, 0xfe64a52ee96b8fc0, + 0xf9bd690a1b68637b, 0x3dfdce7aa3c673b0, + 0x9c1661a651213e2d, 0x6bea10ca65c084e, + 0xc31bfa0fe5698db8, 0x486e494fcff30a62, + 0xf3e2f893dec3f126, 0x5a89dba3c3efccfa, + 0x986ddb5c6b3a76b7, 0xf89629465a75e01c, + 0xbe89523386091465, 0xf6bbb397f1135823, + 0xee2ba6c0678b597f, 0x746aa07ded582e2c, + 0x94db483840b717ef, 0xa8c2a44eb4571cdc, + 0xba121a4650e4ddeb, 0x92f34d62616ce413, + 0xe896a0d7e51e1566, 0x77b020baf9c81d17, + 0x915e2486ef32cd60, 0xace1474dc1d122e, + 0xb5b5ada8aaff80b8, 0xd819992132456ba, + 0xe3231912d5bf60e6, 0x10e1fff697ed6c69, + 0x8df5efabc5979c8f, 0xca8d3ffa1ef463c1, + 0xb1736b96b6fd83b3, 0xbd308ff8a6b17cb2, + 0xddd0467c64bce4a0, 0xac7cb3f6d05ddbde, + 0x8aa22c0dbef60ee4, 0x6bcdf07a423aa96b, + 0xad4ab7112eb3929d, 0x86c16c98d2c953c6, + 0xd89d64d57a607744, 0xe871c7bf077ba8b7, + 0x87625f056c7c4a8b, 0x11471cd764ad4972, + 0xa93af6c6c79b5d2d, 0xd598e40d3dd89bcf, + 0xd389b47879823479, 0x4aff1d108d4ec2c3, + 0x843610cb4bf160cb, 0xcedf722a585139ba, + 0xa54394fe1eedb8fe, 0xc2974eb4ee658828, + 0xce947a3da6a9273e, 0x733d226229feea32, + 0x811ccc668829b887, 0x806357d5a3f525f, + 0xa163ff802a3426a8, 0xca07c2dcb0cf26f7, + 0xc9bcff6034c13052, 0xfc89b393dd02f0b5, + 0xfc2c3f3841f17c67, 0xbbac2078d443ace2, + 0x9d9ba7832936edc0, 0xd54b944b84aa4c0d, + 0xc5029163f384a931, 0xa9e795e65d4df11, + 0xf64335bcf065d37d, 0x4d4617b5ff4a16d5, + 0x99ea0196163fa42e, 0x504bced1bf8e4e45, + 0xc06481fb9bcf8d39, 0xe45ec2862f71e1d6, + 0xf07da27a82c37088, 0x5d767327bb4e5a4c, + 0x964e858c91ba2655, 0x3a6a07f8d510f86f, + 0xbbe226efb628afea, 0x890489f70a55368b, + 0xeadab0aba3b2dbe5, 0x2b45ac74ccea842e, + 0x92c8ae6b464fc96f, 0x3b0b8bc90012929d, + 0xb77ada0617e3bbcb, 0x9ce6ebb40173744, + 0xe55990879ddcaabd, 0xcc420a6a101d0515, + 0x8f57fa54c2a9eab6, 0x9fa946824a12232d, + 0xb32df8e9f3546564, 0x47939822dc96abf9, + 0xdff9772470297ebd, 0x59787e2b93bc56f7, + 0x8bfbea76c619ef36, 0x57eb4edb3c55b65a, + 0xaefae51477a06b03, 0xede622920b6b23f1, + 0xdab99e59958885c4, 0xe95fab368e45eced, + 0x88b402f7fd75539b, 0x11dbcb0218ebb414, + 0xaae103b5fcd2a881, 0xd652bdc29f26a119, + 0xd59944a37c0752a2, 0x4be76d3346f0495f, + 0x857fcae62d8493a5, 0x6f70a4400c562ddb, + 0xa6dfbd9fb8e5b88e, 0xcb4ccd500f6bb952, + 0xd097ad07a71f26b2, 0x7e2000a41346a7a7, + 0x825ecc24c873782f, 0x8ed400668c0c28c8, + 0xa2f67f2dfa90563b, 0x728900802f0f32fa, + 0xcbb41ef979346bca, 0x4f2b40a03ad2ffb9, + 0xfea126b7d78186bc, 0xe2f610c84987bfa8, + 0x9f24b832e6b0f436, 0xdd9ca7d2df4d7c9, + 0xc6ede63fa05d3143, 0x91503d1c79720dbb, + 0xf8a95fcf88747d94, 0x75a44c6397ce912a, + 0x9b69dbe1b548ce7c, 0xc986afbe3ee11aba, + 0xc24452da229b021b, 0xfbe85badce996168, + 0xf2d56790ab41c2a2, 0xfae27299423fb9c3, + 0x97c560ba6b0919a5, 0xdccd879fc967d41a, + 0xbdb6b8e905cb600f, 0x5400e987bbc1c920, + 0xed246723473e3813, 0x290123e9aab23b68, + 0x9436c0760c86e30b, 0xf9a0b6720aaf6521, + 0xb94470938fa89bce, 0xf808e40e8d5b3e69, + 0xe7958cb87392c2c2, 0xb60b1d1230b20e04, + 0x90bd77f3483bb9b9, 0xb1c6f22b5e6f48c2, + 0xb4ecd5f01a4aa828, 0x1e38aeb6360b1af3, + 0xe2280b6c20dd5232, 0x25c6da63c38de1b0, + 0x8d590723948a535f, 0x579c487e5a38ad0e, + 0xb0af48ec79ace837, 0x2d835a9df0c6d851, + 0xdcdb1b2798182244, 0xf8e431456cf88e65, + 0x8a08f0f8bf0f156b, 0x1b8e9ecb641b58ff, + 0xac8b2d36eed2dac5, 0xe272467e3d222f3f, + 0xd7adf884aa879177, 0x5b0ed81dcc6abb0f, + 0x86ccbb52ea94baea, 0x98e947129fc2b4e9, + 0xa87fea27a539e9a5, 0x3f2398d747b36224, + 0xd29fe4b18e88640e, 0x8eec7f0d19a03aad, + 0x83a3eeeef9153e89, 0x1953cf68300424ac, + 0xa48ceaaab75a8e2b, 0x5fa8c3423c052dd7, + 0xcdb02555653131b6, 0x3792f412cb06794d, + 0x808e17555f3ebf11, 0xe2bbd88bbee40bd0, + 0xa0b19d2ab70e6ed6, 0x5b6aceaeae9d0ec4, + 0xc8de047564d20a8b, 0xf245825a5a445275, + 0xfb158592be068d2e, 0xeed6e2f0f0d56712, + 0x9ced737bb6c4183d, 0x55464dd69685606b, + 0xc428d05aa4751e4c, 0xaa97e14c3c26b886, + 0xf53304714d9265df, 0xd53dd99f4b3066a8, + 0x993fe2c6d07b7fab, 0xe546a8038efe4029, + 0xbf8fdb78849a5f96, 0xde98520472bdd033, + 0xef73d256a5c0f77c, 0x963e66858f6d4440, + 0x95a8637627989aad, 0xdde7001379a44aa8, + 0xbb127c53b17ec159, 0x5560c018580d5d52, + 0xe9d71b689dde71af, 0xaab8f01e6e10b4a6, + 0x9226712162ab070d, 0xcab3961304ca70e8, + 0xb6b00d69bb55c8d1, 0x3d607b97c5fd0d22, + 0xe45c10c42a2b3b05, 0x8cb89a7db77c506a, + 0x8eb98a7a9a5b04e3, 0x77f3608e92adb242, + 0xb267ed1940f1c61c, 0x55f038b237591ed3, + 0xdf01e85f912e37a3, 0x6b6c46dec52f6688, + 0x8b61313bbabce2c6, 0x2323ac4b3b3da015, + 0xae397d8aa96c1b77, 0xabec975e0a0d081a, + 0xd9c7dced53c72255, 0x96e7bd358c904a21, + 0x881cea14545c7575, 0x7e50d64177da2e54, + 0xaa242499697392d2, 0xdde50bd1d5d0b9e9, + 0xd4ad2dbfc3d07787, 0x955e4ec64b44e864, + 0x84ec3c97da624ab4, 0xbd5af13bef0b113e, + 0xa6274bbdd0fadd61, 0xecb1ad8aeacdd58e, + 0xcfb11ead453994ba, 0x67de18eda5814af2, + 0x81ceb32c4b43fcf4, 0x80eacf948770ced7, + 0xa2425ff75e14fc31, 0xa1258379a94d028d, + 0xcad2f7f5359a3b3e, 0x96ee45813a04330, + 0xfd87b5f28300ca0d, 0x8bca9d6e188853fc, + 0x9e74d1b791e07e48, 0x775ea264cf55347e, + 0xc612062576589dda, 0x95364afe032a819e, + 0xf79687aed3eec551, 0x3a83ddbd83f52205, + 0x9abe14cd44753b52, 0xc4926a9672793543, + 0xc16d9a0095928a27, 0x75b7053c0f178294, + 0xf1c90080baf72cb1, 0x5324c68b12dd6339, + 0x971da05074da7bee, 0xd3f6fc16ebca5e04, + 0xbce5086492111aea, 0x88f4bb1ca6bcf585, + 0xec1e4a7db69561a5, 0x2b31e9e3d06c32e6, + 0x9392ee8e921d5d07, 0x3aff322e62439fd0, + 0xb877aa3236a4b449, 0x9befeb9fad487c3, + 0xe69594bec44de15b, 0x4c2ebe687989a9b4, + 0x901d7cf73ab0acd9, 0xf9d37014bf60a11, + 0xb424dc35095cd80f, 0x538484c19ef38c95, + 0xe12e13424bb40e13, 0x2865a5f206b06fba, + 0x8cbccc096f5088cb, 0xf93f87b7442e45d4, + 0xafebff0bcb24aafe, 0xf78f69a51539d749, + 0xdbe6fecebdedd5be, 0xb573440e5a884d1c, + 0x89705f4136b4a597, 0x31680a88f8953031, + 0xabcc77118461cefc, 0xfdc20d2b36ba7c3e, + 0xd6bf94d5e57a42bc, 0x3d32907604691b4d, + 0x8637bd05af6c69b5, 0xa63f9a49c2c1b110, + 0xa7c5ac471b478423, 0xfcf80dc33721d54, + 0xd1b71758e219652b, 0xd3c36113404ea4a9, + 0x83126e978d4fdf3b, 0x645a1cac083126ea, + 0xa3d70a3d70a3d70a, 0x3d70a3d70a3d70a4, + 0xcccccccccccccccc, 0xcccccccccccccccd, + 0x8000000000000000, 0x0, + 0xa000000000000000, 0x0, + 0xc800000000000000, 0x0, + 0xfa00000000000000, 0x0, + 0x9c40000000000000, 0x0, + 0xc350000000000000, 0x0, + 0xf424000000000000, 0x0, + 0x9896800000000000, 0x0, + 0xbebc200000000000, 0x0, + 0xee6b280000000000, 0x0, + 0x9502f90000000000, 0x0, + 0xba43b74000000000, 0x0, + 0xe8d4a51000000000, 0x0, + 0x9184e72a00000000, 0x0, + 0xb5e620f480000000, 0x0, + 0xe35fa931a0000000, 0x0, + 0x8e1bc9bf04000000, 0x0, + 0xb1a2bc2ec5000000, 0x0, + 0xde0b6b3a76400000, 0x0, + 0x8ac7230489e80000, 0x0, + 0xad78ebc5ac620000, 0x0, + 0xd8d726b7177a8000, 0x0, + 0x878678326eac9000, 0x0, + 0xa968163f0a57b400, 0x0, + 0xd3c21bcecceda100, 0x0, + 0x84595161401484a0, 0x0, + 0xa56fa5b99019a5c8, 0x0, + 0xcecb8f27f4200f3a, 0x0, + 0x813f3978f8940984, 0x4000000000000000, + 0xa18f07d736b90be5, 0x5000000000000000, + 0xc9f2c9cd04674ede, 0xa400000000000000, + 0xfc6f7c4045812296, 0x4d00000000000000, + 0x9dc5ada82b70b59d, 0xf020000000000000, + 0xc5371912364ce305, 0x6c28000000000000, + 0xf684df56c3e01bc6, 0xc732000000000000, + 0x9a130b963a6c115c, 0x3c7f400000000000, + 0xc097ce7bc90715b3, 0x4b9f100000000000, + 0xf0bdc21abb48db20, 0x1e86d40000000000, + 0x96769950b50d88f4, 0x1314448000000000, + 0xbc143fa4e250eb31, 0x17d955a000000000, + 0xeb194f8e1ae525fd, 0x5dcfab0800000000, + 0x92efd1b8d0cf37be, 0x5aa1cae500000000, + 0xb7abc627050305ad, 0xf14a3d9e40000000, + 0xe596b7b0c643c719, 0x6d9ccd05d0000000, + 0x8f7e32ce7bea5c6f, 0xe4820023a2000000, + 0xb35dbf821ae4f38b, 0xdda2802c8a800000, + 0xe0352f62a19e306e, 0xd50b2037ad200000, + 0x8c213d9da502de45, 0x4526f422cc340000, + 0xaf298d050e4395d6, 0x9670b12b7f410000, + 0xdaf3f04651d47b4c, 0x3c0cdd765f114000, + 0x88d8762bf324cd0f, 0xa5880a69fb6ac800, + 0xab0e93b6efee0053, 0x8eea0d047a457a00, + 0xd5d238a4abe98068, 0x72a4904598d6d880, + 0x85a36366eb71f041, 0x47a6da2b7f864750, + 0xa70c3c40a64e6c51, 0x999090b65f67d924, + 0xd0cf4b50cfe20765, 0xfff4b4e3f741cf6d, + 0x82818f1281ed449f, 0xbff8f10e7a8921a4, + 0xa321f2d7226895c7, 0xaff72d52192b6a0d, + 0xcbea6f8ceb02bb39, 0x9bf4f8a69f764490, + 0xfee50b7025c36a08, 0x2f236d04753d5b4, + 0x9f4f2726179a2245, 0x1d762422c946590, + 0xc722f0ef9d80aad6, 0x424d3ad2b7b97ef5, + 0xf8ebad2b84e0d58b, 0xd2e0898765a7deb2, + 0x9b934c3b330c8577, 0x63cc55f49f88eb2f, + 0xc2781f49ffcfa6d5, 0x3cbf6b71c76b25fb, + 0xf316271c7fc3908a, 0x8bef464e3945ef7a, + 0x97edd871cfda3a56, 0x97758bf0e3cbb5ac, + 0xbde94e8e43d0c8ec, 0x3d52eeed1cbea317, + 0xed63a231d4c4fb27, 0x4ca7aaa863ee4bdd, + 0x945e455f24fb1cf8, 0x8fe8caa93e74ef6a, + 0xb975d6b6ee39e436, 0xb3e2fd538e122b44, + 0xe7d34c64a9c85d44, 0x60dbbca87196b616, + 0x90e40fbeea1d3a4a, 0xbc8955e946fe31cd, + 0xb51d13aea4a488dd, 0x6babab6398bdbe41, + 0xe264589a4dcdab14, 0xc696963c7eed2dd1, + 0x8d7eb76070a08aec, 0xfc1e1de5cf543ca2, + 0xb0de65388cc8ada8, 0x3b25a55f43294bcb, + 0xdd15fe86affad912, 0x49ef0eb713f39ebe, + 0x8a2dbf142dfcc7ab, 0x6e3569326c784337, + 0xacb92ed9397bf996, 0x49c2c37f07965404, + 0xd7e77a8f87daf7fb, 0xdc33745ec97be906, + 0x86f0ac99b4e8dafd, 0x69a028bb3ded71a3, + 0xa8acd7c0222311bc, 0xc40832ea0d68ce0c, + 0xd2d80db02aabd62b, 0xf50a3fa490c30190, + 0x83c7088e1aab65db, 0x792667c6da79e0fa, + 0xa4b8cab1a1563f52, 0x577001b891185938, + 0xcde6fd5e09abcf26, 0xed4c0226b55e6f86, + 0x80b05e5ac60b6178, 0x544f8158315b05b4, + 0xa0dc75f1778e39d6, 0x696361ae3db1c721, + 0xc913936dd571c84c, 0x3bc3a19cd1e38e9, + 0xfb5878494ace3a5f, 0x4ab48a04065c723, + 0x9d174b2dcec0e47b, 0x62eb0d64283f9c76, + 0xc45d1df942711d9a, 0x3ba5d0bd324f8394, + 0xf5746577930d6500, 0xca8f44ec7ee36479, + 0x9968bf6abbe85f20, 0x7e998b13cf4e1ecb, + 0xbfc2ef456ae276e8, 0x9e3fedd8c321a67e, + 0xefb3ab16c59b14a2, 0xc5cfe94ef3ea101e, + 0x95d04aee3b80ece5, 0xbba1f1d158724a12, + 0xbb445da9ca61281f, 0x2a8a6e45ae8edc97, + 0xea1575143cf97226, 0xf52d09d71a3293bd, + 0x924d692ca61be758, 0x593c2626705f9c56, + 0xb6e0c377cfa2e12e, 0x6f8b2fb00c77836c, + 0xe498f455c38b997a, 0xb6dfb9c0f956447, + 0x8edf98b59a373fec, 0x4724bd4189bd5eac, + 0xb2977ee300c50fe7, 0x58edec91ec2cb657, + 0xdf3d5e9bc0f653e1, 0x2f2967b66737e3ed, + 0x8b865b215899f46c, 0xbd79e0d20082ee74, + 0xae67f1e9aec07187, 0xecd8590680a3aa11, + 0xda01ee641a708de9, 0xe80e6f4820cc9495, + 0x884134fe908658b2, 0x3109058d147fdcdd, + 0xaa51823e34a7eede, 0xbd4b46f0599fd415, + 0xd4e5e2cdc1d1ea96, 0x6c9e18ac7007c91a, + 0x850fadc09923329e, 0x3e2cf6bc604ddb0, + 0xa6539930bf6bff45, 0x84db8346b786151c, + 0xcfe87f7cef46ff16, 0xe612641865679a63, + 0x81f14fae158c5f6e, 0x4fcb7e8f3f60c07e, + 0xa26da3999aef7749, 0xe3be5e330f38f09d, + 0xcb090c8001ab551c, 0x5cadf5bfd3072cc5, + 0xfdcb4fa002162a63, 0x73d9732fc7c8f7f6, + 0x9e9f11c4014dda7e, 0x2867e7fddcdd9afa, + 0xc646d63501a1511d, 0xb281e1fd541501b8, + 0xf7d88bc24209a565, 0x1f225a7ca91a4226, + 0x9ae757596946075f, 0x3375788de9b06958, + 0xc1a12d2fc3978937, 0x52d6b1641c83ae, + 0xf209787bb47d6b84, 0xc0678c5dbd23a49a, + 0x9745eb4d50ce6332, 0xf840b7ba963646e0, + 0xbd176620a501fbff, 0xb650e5a93bc3d898, + 0xec5d3fa8ce427aff, 0xa3e51f138ab4cebe, + 0x93ba47c980e98cdf, 0xc66f336c36b10137, + 0xb8a8d9bbe123f017, 0xb80b0047445d4184, + 0xe6d3102ad96cec1d, 0xa60dc059157491e5, + 0x9043ea1ac7e41392, 0x87c89837ad68db2f, + 0xb454e4a179dd1877, 0x29babe4598c311fb, + 0xe16a1dc9d8545e94, 0xf4296dd6fef3d67a, + 0x8ce2529e2734bb1d, 0x1899e4a65f58660c, + 0xb01ae745b101e9e4, 0x5ec05dcff72e7f8f, + 0xdc21a1171d42645d, 0x76707543f4fa1f73, + 0x899504ae72497eba, 0x6a06494a791c53a8, + 0xabfa45da0edbde69, 0x487db9d17636892, + 0xd6f8d7509292d603, 0x45a9d2845d3c42b6, + 0x865b86925b9bc5c2, 0xb8a2392ba45a9b2, + 0xa7f26836f282b732, 0x8e6cac7768d7141e, + 0xd1ef0244af2364ff, 0x3207d795430cd926, + 0x8335616aed761f1f, 0x7f44e6bd49e807b8, + 0xa402b9c5a8d3a6e7, 0x5f16206c9c6209a6, + 0xcd036837130890a1, 0x36dba887c37a8c0f, + 0x802221226be55a64, 0xc2494954da2c9789, + 0xa02aa96b06deb0fd, 0xf2db9baa10b7bd6c, + 0xc83553c5c8965d3d, 0x6f92829494e5acc7, + 0xfa42a8b73abbf48c, 0xcb772339ba1f17f9, + 0x9c69a97284b578d7, 0xff2a760414536efb, + 0xc38413cf25e2d70d, 0xfef5138519684aba, + 0xf46518c2ef5b8cd1, 0x7eb258665fc25d69, + 0x98bf2f79d5993802, 0xef2f773ffbd97a61, + 0xbeeefb584aff8603, 0xaafb550ffacfd8fa, + 0xeeaaba2e5dbf6784, 0x95ba2a53f983cf38, + 0x952ab45cfa97a0b2, 0xdd945a747bf26183, + 0xba756174393d88df, 0x94f971119aeef9e4, + 0xe912b9d1478ceb17, 0x7a37cd5601aab85d, + 0x91abb422ccb812ee, 0xac62e055c10ab33a, + 0xb616a12b7fe617aa, 0x577b986b314d6009, + 0xe39c49765fdf9d94, 0xed5a7e85fda0b80b, + 0x8e41ade9fbebc27d, 0x14588f13be847307, + 0xb1d219647ae6b31c, 0x596eb2d8ae258fc8, + 0xde469fbd99a05fe3, 0x6fca5f8ed9aef3bb, + 0x8aec23d680043bee, 0x25de7bb9480d5854, + 0xada72ccc20054ae9, 0xaf561aa79a10ae6a, + 0xd910f7ff28069da4, 0x1b2ba1518094da04, + 0x87aa9aff79042286, 0x90fb44d2f05d0842, + 0xa99541bf57452b28, 0x353a1607ac744a53, + 0xd3fa922f2d1675f2, 0x42889b8997915ce8, + 0x847c9b5d7c2e09b7, 0x69956135febada11, + 0xa59bc234db398c25, 0x43fab9837e699095, + 0xcf02b2c21207ef2e, 0x94f967e45e03f4bb, + 0x8161afb94b44f57d, 0x1d1be0eebac278f5, + 0xa1ba1ba79e1632dc, 0x6462d92a69731732, + 0xca28a291859bbf93, 0x7d7b8f7503cfdcfe, + 0xfcb2cb35e702af78, 0x5cda735244c3d43e, + 0x9defbf01b061adab, 0x3a0888136afa64a7, + 0xc56baec21c7a1916, 0x88aaa1845b8fdd0, + 0xf6c69a72a3989f5b, 0x8aad549e57273d45, + 0x9a3c2087a63f6399, 0x36ac54e2f678864b, + 0xc0cb28a98fcf3c7f, 0x84576a1bb416a7dd, + 0xf0fdf2d3f3c30b9f, 0x656d44a2a11c51d5, + 0x969eb7c47859e743, 0x9f644ae5a4b1b325, + 0xbc4665b596706114, 0x873d5d9f0dde1fee, + 0xeb57ff22fc0c7959, 0xa90cb506d155a7ea, + 0x9316ff75dd87cbd8, 0x9a7f12442d588f2, + 0xb7dcbf5354e9bece, 0xc11ed6d538aeb2f, + 0xe5d3ef282a242e81, 0x8f1668c8a86da5fa, + 0x8fa475791a569d10, 0xf96e017d694487bc, + 0xb38d92d760ec4455, 0x37c981dcc395a9ac, + 0xe070f78d3927556a, 0x85bbe253f47b1417, + 0x8c469ab843b89562, 0x93956d7478ccec8e, + 0xaf58416654a6babb, 0x387ac8d1970027b2, + 0xdb2e51bfe9d0696a, 0x6997b05fcc0319e, + 0x88fcf317f22241e2, 0x441fece3bdf81f03, + 0xab3c2fddeeaad25a, 0xd527e81cad7626c3, + 0xd60b3bd56a5586f1, 0x8a71e223d8d3b074, + 0x85c7056562757456, 0xf6872d5667844e49, + 0xa738c6bebb12d16c, 0xb428f8ac016561db, + 0xd106f86e69d785c7, 0xe13336d701beba52, + 0x82a45b450226b39c, 0xecc0024661173473, + 0xa34d721642b06084, 0x27f002d7f95d0190, + 0xcc20ce9bd35c78a5, 0x31ec038df7b441f4, + 0xff290242c83396ce, 0x7e67047175a15271, + 0x9f79a169bd203e41, 0xf0062c6e984d386, + 0xc75809c42c684dd1, 0x52c07b78a3e60868, + 0xf92e0c3537826145, 0xa7709a56ccdf8a82, + 0x9bbcc7a142b17ccb, 0x88a66076400bb691, + 0xc2abf989935ddbfe, 0x6acff893d00ea435, + 0xf356f7ebf83552fe, 0x583f6b8c4124d43, + 0x98165af37b2153de, 0xc3727a337a8b704a, + 0xbe1bf1b059e9a8d6, 0x744f18c0592e4c5c, + 0xeda2ee1c7064130c, 0x1162def06f79df73, + 0x9485d4d1c63e8be7, 0x8addcb5645ac2ba8, + 0xb9a74a0637ce2ee1, 0x6d953e2bd7173692, + 0xe8111c87c5c1ba99, 0xc8fa8db6ccdd0437, + 0x910ab1d4db9914a0, 0x1d9c9892400a22a2, + 0xb54d5e4a127f59c8, 0x2503beb6d00cab4b, + 0xe2a0b5dc971f303a, 0x2e44ae64840fd61d, + 0x8da471a9de737e24, 0x5ceaecfed289e5d2, + 0xb10d8e1456105dad, 0x7425a83e872c5f47, + 0xdd50f1996b947518, 0xd12f124e28f77719, + 0x8a5296ffe33cc92f, 0x82bd6b70d99aaa6f, + 0xace73cbfdc0bfb7b, 0x636cc64d1001550b, + 0xd8210befd30efa5a, 0x3c47f7e05401aa4e, + 0x8714a775e3e95c78, 0x65acfaec34810a71, + 0xa8d9d1535ce3b396, 0x7f1839a741a14d0d, + 0xd31045a8341ca07c, 0x1ede48111209a050, + 0x83ea2b892091e44d, 0x934aed0aab460432, + 0xa4e4b66b68b65d60, 0xf81da84d5617853f, + 0xce1de40642e3f4b9, 0x36251260ab9d668e, + 0x80d2ae83e9ce78f3, 0xc1d72b7c6b426019, + 0xa1075a24e4421730, 0xb24cf65b8612f81f, + 0xc94930ae1d529cfc, 0xdee033f26797b627, + 0xfb9b7cd9a4a7443c, 0x169840ef017da3b1, + 0x9d412e0806e88aa5, 0x8e1f289560ee864e, + 0xc491798a08a2ad4e, 0xf1a6f2bab92a27e2, + 0xf5b5d7ec8acb58a2, 0xae10af696774b1db, + 0x9991a6f3d6bf1765, 0xacca6da1e0a8ef29, + 0xbff610b0cc6edd3f, 0x17fd090a58d32af3, + 0xeff394dcff8a948e, 0xddfc4b4cef07f5b0, + 0x95f83d0a1fb69cd9, 0x4abdaf101564f98e, + 0xbb764c4ca7a4440f, 0x9d6d1ad41abe37f1, + 0xea53df5fd18d5513, 0x84c86189216dc5ed, + 0x92746b9be2f8552c, 0x32fd3cf5b4e49bb4, + 0xb7118682dbb66a77, 0x3fbc8c33221dc2a1, + 0xe4d5e82392a40515, 0xfabaf3feaa5334a, + 0x8f05b1163ba6832d, 0x29cb4d87f2a7400e, + 0xb2c71d5bca9023f8, 0x743e20e9ef511012, + 0xdf78e4b2bd342cf6, 0x914da9246b255416, + 0x8bab8eefb6409c1a, 0x1ad089b6c2f7548e, + 0xae9672aba3d0c320, 0xa184ac2473b529b1, + 0xda3c0f568cc4f3e8, 0xc9e5d72d90a2741e, + 0x8865899617fb1871, 0x7e2fa67c7a658892, + 0xaa7eebfb9df9de8d, 0xddbb901b98feeab7, + 0xd51ea6fa85785631, 0x552a74227f3ea565, + 0x8533285c936b35de, 0xd53a88958f87275f, + 0xa67ff273b8460356, 0x8a892abaf368f137, + 0xd01fef10a657842c, 0x2d2b7569b0432d85, + 0x8213f56a67f6b29b, 0x9c3b29620e29fc73, + 0xa298f2c501f45f42, 0x8349f3ba91b47b8f, + 0xcb3f2f7642717713, 0x241c70a936219a73, + 0xfe0efb53d30dd4d7, 0xed238cd383aa0110, + 0x9ec95d1463e8a506, 0xf4363804324a40aa, + 0xc67bb4597ce2ce48, 0xb143c6053edcd0d5, + 0xf81aa16fdc1b81da, 0xdd94b7868e94050a, + 0x9b10a4e5e9913128, 0xca7cf2b4191c8326, + 0xc1d4ce1f63f57d72, 0xfd1c2f611f63a3f0, + 0xf24a01a73cf2dccf, 0xbc633b39673c8cec, + 0x976e41088617ca01, 0xd5be0503e085d813, + 0xbd49d14aa79dbc82, 0x4b2d8644d8a74e18, + 0xec9c459d51852ba2, 0xddf8e7d60ed1219e, + 0x93e1ab8252f33b45, 0xcabb90e5c942b503, + 0xb8da1662e7b00a17, 0x3d6a751f3b936243, + 0xe7109bfba19c0c9d, 0xcc512670a783ad4, + 0x906a617d450187e2, 0x27fb2b80668b24c5, + 0xb484f9dc9641e9da, 0xb1f9f660802dedf6, + 0xe1a63853bbd26451, 0x5e7873f8a0396973, + 0x8d07e33455637eb2, 0xdb0b487b6423e1e8, + 0xb049dc016abc5e5f, 0x91ce1a9a3d2cda62, + 0xdc5c5301c56b75f7, 0x7641a140cc7810fb, + 0x89b9b3e11b6329ba, 0xa9e904c87fcb0a9d, + 0xac2820d9623bf429, 0x546345fa9fbdcd44, + 0xd732290fbacaf133, 0xa97c177947ad4095, + 0x867f59a9d4bed6c0, 0x49ed8eabcccc485d, + 0xa81f301449ee8c70, 0x5c68f256bfff5a74, + 0xd226fc195c6a2f8c, 0x73832eec6fff3111, + 0x83585d8fd9c25db7, 0xc831fd53c5ff7eab, + 0xa42e74f3d032f525, 0xba3e7ca8b77f5e55, + 0xcd3a1230c43fb26f, 0x28ce1bd2e55f35eb, + 0x80444b5e7aa7cf85, 0x7980d163cf5b81b3, + 0xa0555e361951c366, 0xd7e105bcc332621f, + 0xc86ab5c39fa63440, 0x8dd9472bf3fefaa7, + 0xfa856334878fc150, 0xb14f98f6f0feb951, + 0x9c935e00d4b9d8d2, 0x6ed1bf9a569f33d3, + 0xc3b8358109e84f07, 0xa862f80ec4700c8, + 0xf4a642e14c6262c8, 0xcd27bb612758c0fa, + 0x98e7e9cccfbd7dbd, 0x8038d51cb897789c, + 0xbf21e44003acdd2c, 0xe0470a63e6bd56c3, + 0xeeea5d5004981478, 0x1858ccfce06cac74, + 0x95527a5202df0ccb, 0xf37801e0c43ebc8, + 0xbaa718e68396cffd, 0xd30560258f54e6ba, + 0xe950df20247c83fd, 0x47c6b82ef32a2069, + 0x91d28b7416cdd27e, 0x4cdc331d57fa5441, + 0xb6472e511c81471d, 0xe0133fe4adf8e952, + 0xe3d8f9e563a198e5, 0x58180fddd97723a6, + 0x8e679c2f5e44ff8f, 0x570f09eaa7ea7648, + }; +}; + +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE + +template +constexpr uint64_t + powers_template::power_of_five_128[number_of_entries]; + +#endif + +using powers = powers_template<>; + +} // namespace simdjson_fast_float + +#endif + +#ifndef SIMDJSON_FASTFLOAT_DECIMAL_TO_BINARY_H +#define SIMDJSON_FASTFLOAT_DECIMAL_TO_BINARY_H + +#include +#include +#include +#include +#include +#include + +namespace simdjson_fast_float { + +// This will compute or rather approximate w * 5**q and return a pair of 64-bit +// words approximating the result, with the "high" part corresponding to the +// most significant bits and the low part corresponding to the least significant +// bits. +// +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 value128 +compute_product_approximation(int64_t q, uint64_t w) { + int const index = 2 * int(q - powers::smallest_power_of_five); + // For small values of q, e.g., q in [0,27], the answer is always exact + // because The line value128 firstproduct = full_multiplication(w, + // power_of_five_128[index]); gives the exact answer. + value128 firstproduct = + full_multiplication(w, powers::power_of_five_128[index]); + static_assert((bit_precision >= 0) && (bit_precision <= 64), + " precision should be in (0,64]"); + constexpr uint64_t precision_mask = + (bit_precision < 64) ? (uint64_t(0xFFFFFFFFFFFFFFFF) >> bit_precision) + : uint64_t(0xFFFFFFFFFFFFFFFF); + if ((firstproduct.high & precision_mask) == + precision_mask) { // could further guard with (lower + w < lower) + // regarding the second product, we only need secondproduct.high, but our + // expectation is that the compiler will optimize this extra work away if + // needed. + value128 secondproduct = + full_multiplication(w, powers::power_of_five_128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { + firstproduct.high++; + } + } + return firstproduct; +} + +namespace detail { +/** + * For q in (0,350), we have that + * f = (((152170 + 65536) * q ) >> 16); + * is equal to + * floor(p) + q + * where + * p = log(5**q)/log(2) = q * log(5)/log(2) + * + * For negative values of q in (-400,0), we have that + * f = (((152170 + 65536) * q ) >> 16); + * is equal to + * -ceil(p) + q + * where + * p = log(5**-q)/log(2) = -q * log(5)/log(2) + */ +constexpr simdjson_fastfloat_really_inline int32_t power(int32_t q) noexcept { + return (((152170 + 65536) * q) >> 16) + 63; +} +} // namespace detail + +// create an adjusted mantissa, biased by the invalid power2 +// for significant digits already multiplied by 10 ** q. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 adjusted_mantissa +compute_error_scaled(int64_t q, uint64_t w, int lz) noexcept { + int hilz = int(w >> 63) ^ 1; + adjusted_mantissa answer; + answer.mantissa = w << hilz; + int bias = binary::mantissa_explicit_bits() - binary::minimum_exponent(); + answer.power2 = int32_t(detail::power(int32_t(q)) + bias - hilz - lz - 62 + + invalid_am_bias); + return answer; +} + +// w * 10 ** q, without rounding the representation up. +// the power2 in the exponent will be adjusted by invalid_am_bias. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +compute_error(int64_t q, uint64_t w) noexcept { + int lz = leading_zeroes(w); + w <<= lz; + value128 product = + compute_product_approximation(q, w); + return compute_error_scaled(q, product.high, lz); +} + +// Computers w * 10 ** q. +// The returned value should be a valid number that simply needs to be +// packed. However, in some very rare cases, the computation will fail. In such +// cases, we return an adjusted_mantissa with a negative power of 2: the caller +// should recompute in such cases. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +compute_float(int64_t q, uint64_t w) noexcept { + adjusted_mantissa answer; + if ((w == 0) || (q < binary::smallest_power_of_ten())) { + answer.power2 = 0; + answer.mantissa = 0; + // result should be zero + return answer; + } + if (q > binary::largest_power_of_ten()) { + // we want to get infinity: + answer.power2 = binary::infinite_power(); + answer.mantissa = 0; + return answer; + } + // At this point in time q is in [powers::smallest_power_of_five, + // powers::largest_power_of_five]. + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(w); + w <<= lz; + + // The required precision is binary::mantissa_explicit_bits() + 3 because + // 1. We need the implicit bit + // 2. We need an extra bit for rounding purposes + // 3. We might lose a bit due to the "upperbit" routine (result too small, + // requiring a shift) + + value128 product = + compute_product_approximation(q, w); + // The computed 'product' is always sufficient. + // Mathematical proof: + // Noble Mushtak and Daniel Lemire, Fast Number Parsing Without Fallback (to + // appear) See script/mushtak_lemire.py + + // The "compute_product_approximation" function can be slightly slower than a + // branchless approach: value128 product = compute_product(q, w); but in + // practice, we can win big with the compute_product_approximation if its + // additional branch is easily predicted. Which is best is data specific. + int upperbit = int(product.high >> 63); + int shift = upperbit + 64 - binary::mantissa_explicit_bits() - 3; + + answer.mantissa = product.high >> shift; + + answer.power2 = int32_t(detail::power(int32_t(q)) + upperbit - lz - + binary::minimum_exponent()); + if (answer.power2 <= 0) { // we have a subnormal? + // Here have that answer.power2 <= 0 so -answer.power2 >= 0 + if (-answer.power2 + 1 >= + 64) { // if we have more than 64 bits below the minimum exponent, you + // have a zero for sure. + answer.power2 = 0; + answer.mantissa = 0; + // result should be zero + return answer; + } + // next line is safe because -answer.power2 + 1 < 64 + answer.mantissa >>= -answer.power2 + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0 in the 32-bit and + // and 64-bit case (with no more than 19 digits). + answer.mantissa += (answer.mantissa & 1); // round up + answer.mantissa >>= 1; + // There is a weird scenario where we don't have a subnormal but just. + // Suppose we start with 2.2250738585072013e-308, we end up + // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + // subnormal, but we can only know this after rounding. + // So we only declare a subnormal if we are smaller than the threshold. + answer.power2 = + (answer.mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) + ? 0 + : 1; + return answer; + } + + // usually, we round *up*, but if we fall right in between and and we have an + // even basis, we need to round down + // We are only concerned with the cases where 5**q fits in single 64-bit word. + if ((product.low <= 1) && (q >= binary::min_exponent_round_to_even()) && + (q <= binary::max_exponent_round_to_even()) && + ((answer.mantissa & 3) == 1)) { // we may fall between two floats! + // To be in-between two floats we need that in doing + // answer.mantissa = product.high >> (upperbit + 64 - + // binary::mantissa_explicit_bits() - 3); + // ... we dropped out only zeroes. But if this happened, then we can go + // back!!! + if ((answer.mantissa << shift) == product.high) { + answer.mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + answer.mantissa += (answer.mantissa & 1); // round up + answer.mantissa >>= 1; + if (answer.mantissa >= (uint64_t(2) << binary::mantissa_explicit_bits())) { + answer.mantissa = (uint64_t(1) << binary::mantissa_explicit_bits()); + answer.power2++; // undo previous addition + } + + answer.mantissa &= ~(uint64_t(1) << binary::mantissa_explicit_bits()); + if (answer.power2 >= binary::infinite_power()) { // infinity + answer.power2 = binary::infinite_power(); + answer.mantissa = 0; + } + return answer; +} + +} // namespace simdjson_fast_float + +#endif + +#ifndef SIMDJSON_FASTFLOAT_BIGINT_H +#define SIMDJSON_FASTFLOAT_BIGINT_H + +#include +#include +#include +#include + + +namespace simdjson_fast_float { + +// the limb width: we want efficient multiplication of double the bits in +// limb, or for 64-bit limbs, at least 64-bit multiplication where we can +// extract the high and low parts efficiently. this is every 64-bit +// architecture except for sparc, which emulates 128-bit multiplication. +// we might have platforms where `CHAR_BIT` is not 8, so let's avoid +// doing `8 * sizeof(limb)`. +#if defined(SIMDJSON_FASTFLOAT_64BIT) && !defined(__sparc) +#define SIMDJSON_FASTFLOAT_64BIT_LIMB 1 +typedef uint64_t limb; +constexpr size_t limb_bits = 64; +#else +#define SIMDJSON_FASTFLOAT_32BIT_LIMB +typedef uint32_t limb; +constexpr size_t limb_bits = 32; +#endif + +typedef span limb_span; + +// number of bits in a bigint. this needs to be at least the number +// of bits required to store the largest bigint, which is +// `log2(10**(digits + max_exp))`, or `log2(10**(767 + 342))`, or +// ~3600 bits, so we round to 4000. +constexpr size_t bigint_bits = 4000; +constexpr size_t bigint_limbs = bigint_bits / limb_bits; + +// vector-like type that is allocated on the stack. the entire +// buffer is pre-allocated, and only the length changes. +template struct stackvec { + limb data[size]; + // we never need more than 150 limbs + uint16_t length{0}; + + stackvec() = default; + stackvec(stackvec const &) = delete; + stackvec &operator=(stackvec const &) = delete; + stackvec(stackvec &&) = delete; + stackvec &operator=(stackvec &&other) = delete; + + // create stack vector from existing limb span. + SIMDJSON_FASTFLOAT_CONSTEXPR20 stackvec(limb_span s) { + SIMDJSON_FASTFLOAT_ASSERT(try_extend(s)); + } + + SIMDJSON_FASTFLOAT_CONSTEXPR14 limb &operator[](size_t index) noexcept { + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); + return data[index]; + } + + SIMDJSON_FASTFLOAT_CONSTEXPR14 const limb &operator[](size_t index) const noexcept { + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); + return data[index]; + } + + // index from the end of the container + SIMDJSON_FASTFLOAT_CONSTEXPR14 const limb &rindex(size_t index) const noexcept { + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(index < length); + size_t rindex = length - index - 1; + return data[rindex]; + } + + // set the length, without bounds checking. + SIMDJSON_FASTFLOAT_CONSTEXPR14 void set_len(size_t len) noexcept { + length = uint16_t(len); + } + + constexpr size_t len() const noexcept { return length; } + + constexpr bool is_empty() const noexcept { return length == 0; } + + constexpr size_t capacity() const noexcept { return size; } + + // append item to vector, without bounds checking + SIMDJSON_FASTFLOAT_CONSTEXPR14 void push_unchecked(limb value) noexcept { + data[length] = value; + length++; + } + + // append item to vector, returning if item was added + SIMDJSON_FASTFLOAT_CONSTEXPR14 bool try_push(limb value) noexcept { + if (len() < capacity()) { + push_unchecked(value); + return true; + } else { + return false; + } + } + + // add items to the vector, from a span, without bounds checking + SIMDJSON_FASTFLOAT_CONSTEXPR20 void extend_unchecked(limb_span s) noexcept { + limb *ptr = data + length; + std::copy_n(s.ptr, s.len(), ptr); + set_len(len() + s.len()); + } + + // try to add items to the vector, returning if items were added + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool try_extend(limb_span s) noexcept { + if (len() + s.len() <= capacity()) { + extend_unchecked(s); + return true; + } else { + return false; + } + } + + // resize the vector, without bounds checking + // if the new size is longer than the vector, assign value to each + // appended item. + SIMDJSON_FASTFLOAT_CONSTEXPR20 + void resize_unchecked(size_t new_len, limb value) noexcept { + if (new_len > len()) { + size_t count = new_len - len(); + limb *first = data + len(); + limb *last = first + count; + ::std::fill(first, last, value); + set_len(new_len); + } else { + set_len(new_len); + } + } + + // try to resize the vector, returning if the vector was resized. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool try_resize(size_t new_len, limb value) noexcept { + if (new_len > capacity()) { + return false; + } else { + resize_unchecked(new_len, value); + return true; + } + } + + // check if any limbs are non-zero after the given index. + // this needs to be done in reverse order, since the index + // is relative to the most significant limbs. + SIMDJSON_FASTFLOAT_CONSTEXPR14 bool nonzero(size_t index) const noexcept { + while (index < len()) { + if (rindex(index) != 0) { + return true; + } + index++; + } + return false; + } + + // normalize the big integer, so most-significant zero limbs are removed. + SIMDJSON_FASTFLOAT_CONSTEXPR14 void normalize() noexcept { + while (len() > 0 && rindex(0) == 0) { + length--; + } + } +}; + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 uint64_t +empty_hi64(bool &truncated) noexcept { + truncated = false; + return 0; +} - std::memmove(buf + (2 + static_cast(-n)), buf, - static_cast(k)); - buf[0] = '0'; - buf[1] = '.'; - std::memset(buf + 2, '0', static_cast(-n)); - return buf + (2U + static_cast(-n) + static_cast(k)); +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +uint64_hi64(uint64_t r0, bool &truncated) noexcept { + truncated = false; + int shl = leading_zeroes(r0); + return r0 << shl; +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +uint64_hi64(uint64_t r0, uint64_t r1, bool &truncated) noexcept { + int shl = leading_zeroes(r0); + if (shl == 0) { + truncated = r1 != 0; + return r0; + } else { + int shr = 64 - shl; + truncated = (r1 << shl) != 0; + return (r0 << shl) | (r1 >> shr); } +} - if (k == 1) { - // dE+123 - // len <= 1 + 5 +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +uint32_hi64(uint32_t r0, bool &truncated) noexcept { + return uint64_hi64(r0, truncated); +} - buf += 1; - } else { - // d.igitsE+123 - // len <= max_digits10 + 1 + 5 +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +uint32_hi64(uint32_t r0, uint32_t r1, bool &truncated) noexcept { + uint64_t x0 = r0; + uint64_t x1 = r1; + return uint64_hi64((x0 << 32) | x1, truncated); +} - std::memmove(buf + 2, buf + 1, static_cast(k) - 1); - buf[1] = '.'; - buf += 1 + static_cast(k); +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t +uint32_hi64(uint32_t r0, uint32_t r1, uint32_t r2, bool &truncated) noexcept { + uint64_t x0 = r0; + uint64_t x1 = r1; + uint64_t x2 = r2; + return uint64_hi64(x0, (x1 << 32) | x2, truncated); +} + +// add two small integers, checking for overflow. +// we want an efficient operation. for msvc, where +// we don't have built-in intrinsics, this is still +// pretty fast. +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 limb +scalar_add(limb x, limb y, bool &overflow) noexcept { + limb z; +// gcc and clang +#if defined(__has_builtin) +#if __has_builtin(__builtin_add_overflow) + if (!cpp20_and_in_constexpr()) { + overflow = __builtin_add_overflow(x, y, &z); + return z; } +#endif +#endif - *buf++ = 'e'; - return append_exponent(buf, n - 1); + // generic, this still optimizes correctly on MSVC. + z = x + y; + overflow = z < x; + return z; +} + +// multiply two small integers, getting both the high and low bits. +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 limb +scalar_mul(limb x, limb y, limb &carry) noexcept { +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB +#if defined(__SIZEOF_INT128__) + // GCC and clang both define it as an extension. + __uint128_t z = __uint128_t(x) * __uint128_t(y) + __uint128_t(carry); + carry = limb(z >> limb_bits); + return limb(z); +#else + // fallback, no native 128-bit integer multiplication with carry. + // on msvc, this optimizes identically, somehow. + value128 z = full_multiplication(x, y); + bool overflow; + z.low = scalar_add(z.low, carry, overflow); + z.high += uint64_t(overflow); // cannot overflow + carry = z.high; + return z.low; +#endif +#else + uint64_t z = uint64_t(x) * uint64_t(y) + uint64_t(carry); + carry = limb(z >> limb_bits); + return limb(z); +#endif } -} // namespace dtoa_impl +// add scalar value to bigint starting from offset. +// used in grade school multiplication +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool small_add_from(stackvec &vec, limb y, + size_t start) noexcept { + size_t index = start; + limb carry = y; + bool overflow; + while (carry != 0 && index < vec.len()) { + vec[index] = scalar_add(vec[index], carry, overflow); + carry = limb(overflow); + index += 1; + } + if (carry != 0) { + SIMDJSON_FASTFLOAT_TRY(vec.try_push(carry)); + } + return true; +} -/*! -The format of the resulting decimal representation is similar to printf's %g -format. Returns an iterator pointing past-the-end of the decimal representation. -@note The input number must be finite, i.e. NaN's and Inf's are not supported. -@note The buffer must be large enough. -@note The result is NOT null-terminated. -*/ -char *to_chars(char *first, const char *last, double value) { - static_cast(last); // maybe unused - fix warning - bool negative = std::signbit(value); - if (negative) { - value = -value; - *first++ = '-'; +// add scalar value to bigint. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +small_add(stackvec &vec, limb y) noexcept { + return small_add_from(vec, y, 0); +} + +// multiply bigint by scalar value. +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool small_mul(stackvec &vec, + limb y) noexcept { + limb carry = 0; + for (size_t index = 0; index < vec.len(); index++) { + vec[index] = scalar_mul(vec[index], y, carry); + } + if (carry != 0) { + SIMDJSON_FASTFLOAT_TRY(vec.try_push(carry)); } + return true; +} - if (value == 0) // +-0 - { - *first++ = '0'; - // Make it look like a floating-point number (#362, #378) - *first++ = '.'; - *first++ = '0'; - return first; +// add bigint to bigint starting from index. +// used in grade school multiplication +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 bool large_add_from(stackvec &x, limb_span y, + size_t start) noexcept { + // the effective x buffer is from `xstart..x.len()`, so exit early + // if we can't get that current range. + if (x.len() < start || y.len() > x.len() - start) { + SIMDJSON_FASTFLOAT_TRY(x.try_resize(y.len() + start, 0)); } - // Compute v = buffer * 10^decimal_exponent. - // The decimal digits are stored in the buffer, which needs to be interpreted - // as an unsigned decimal integer. - // len is the length of the buffer, i.e. the number of decimal digits. - int len = 0; - int decimal_exponent = 0; - dtoa_impl::grisu2(first, len, decimal_exponent, value); - // Format the buffer like printf("%.*g", prec, value) - constexpr int kMinExp = -4; - constexpr int kMaxExp = std::numeric_limits::digits10; - return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, - kMaxExp); + bool carry = false; + for (size_t index = 0; index < y.len(); index++) { + limb xi = x[index + start]; + limb yi = y[index]; + bool c1 = false; + bool c2 = false; + xi = scalar_add(xi, yi, c1); + if (carry) { + xi = scalar_add(xi, 1, c2); + } + x[index + start] = xi; + carry = c1 | c2; + } + + // handle overflow + if (carry) { + SIMDJSON_FASTFLOAT_TRY(small_add_from(x, 1, y.len() + start)); + } + return true; } -} // namespace internal -} // namespace simdjson -#endif // SIMDJSON_SRC_TO_CHARS_CPP -/* end file to_chars.cpp */ -/* including from_chars.cpp: #include */ -/* begin file from_chars.cpp */ -#ifndef SIMDJSON_SRC_FROM_CHARS_CPP -#define SIMDJSON_SRC_FROM_CHARS_CPP +// add bigint to bigint. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +large_add_from(stackvec &x, limb_span y) noexcept { + return large_add_from(x, y, 0); +} + +// grade-school multiplication algorithm +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 bool long_mul(stackvec &x, limb_span y) noexcept { + limb_span xs = limb_span(x.data, x.len()); + stackvec z(xs); + limb_span zs = limb_span(z.data, z.len()); + + if (y.len() != 0) { + limb y0 = y[0]; + SIMDJSON_FASTFLOAT_TRY(small_mul(x, y0)); + for (size_t index = 1; index < y.len(); index++) { + limb yi = y[index]; + stackvec zi; + if (yi != 0) { + // re-use the same buffer throughout + zi.set_len(0); + SIMDJSON_FASTFLOAT_TRY(zi.try_extend(zs)); + SIMDJSON_FASTFLOAT_TRY(small_mul(zi, yi)); + limb_span zis = limb_span(zi.data, zi.len()); + SIMDJSON_FASTFLOAT_TRY(large_add_from(x, zis, index)); + } + } + } -/* skipped duplicate #include */ + x.normalize(); + return true; +} -#include -#include -#include +// grade-school multiplication algorithm +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 bool large_mul(stackvec &x, limb_span y) noexcept { + if (y.len() == 1) { + SIMDJSON_FASTFLOAT_TRY(small_mul(x, y[0])); + } else { + SIMDJSON_FASTFLOAT_TRY(long_mul(x, y)); + } + return true; +} -namespace simdjson { -namespace internal { +template struct pow5_tables { + static constexpr uint32_t large_step = 135; + static constexpr uint64_t small_power_of_5[] = { + 1UL, + 5UL, + 25UL, + 125UL, + 625UL, + 3125UL, + 15625UL, + 78125UL, + 390625UL, + 1953125UL, + 9765625UL, + 48828125UL, + 244140625UL, + 1220703125UL, + 6103515625UL, + 30517578125UL, + 152587890625UL, + 762939453125UL, + 3814697265625UL, + 19073486328125UL, + 95367431640625UL, + 476837158203125UL, + 2384185791015625UL, + 11920928955078125UL, + 59604644775390625UL, + 298023223876953125UL, + 1490116119384765625UL, + 7450580596923828125UL, + }; +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + constexpr static limb large_power_of_5[] = { + 1414648277510068013UL, 9180637584431281687UL, 4539964771860779200UL, + 10482974169319127550UL, 198276706040285095UL}; +#else + constexpr static limb large_power_of_5[] = { + 4279965485U, 329373468U, 4020270615U, 2137533757U, 4287402176U, + 1057042919U, 1071430142U, 2440757623U, 381945767U, 46164893U}; +#endif +}; -/** - * The code in the internal::from_chars function is meant to handle the floating-point number parsing - * when we have more than 19 digits in the decimal mantissa. This should only be seen - * in adversarial scenarios: we do not expect production systems to even produce - * such floating-point numbers. - * - * The parser is based on work by Nigel Tao (at https://github.com/google/wuffs/) - * who credits Ken Thompson for the design (via a reference to the Go source - * code). See - * https://github.com/google/wuffs/blob/aa46859ea40c72516deffa1b146121952d6dfd3b/internal/cgen/base/floatconv-submodule-data.c - * https://github.com/google/wuffs/blob/46cd8105f47ca07ae2ba8e6a7818ef9c0df6c152/internal/cgen/base/floatconv-submodule-code.c - * It is probably not very fast but it is a fallback that should almost never be - * called in real life. Google Wuffs is published under APL 2.0. - **/ +#if SIMDJSON_FASTFLOAT_DETAIL_MUST_DEFINE_CONSTEXPR_VARIABLE -namespace { -constexpr uint32_t max_digits = 768; -constexpr int32_t decimal_point_range = 2047; -} // namespace +template constexpr uint32_t pow5_tables::large_step; -struct adjusted_mantissa { - uint64_t mantissa; - int power2; - adjusted_mantissa() : mantissa(0), power2(0) {} -}; +template constexpr uint64_t pow5_tables::small_power_of_5[]; -struct decimal { - uint32_t num_digits; - int32_t decimal_point; - bool negative; - bool truncated; - uint8_t digits[max_digits]; -}; +template constexpr limb pow5_tables::large_power_of_5[]; -template struct binary_format { - static constexpr int mantissa_explicit_bits(); - static constexpr int minimum_exponent(); - static constexpr int infinite_power(); - static constexpr int sign_index(); -}; +#endif -template <> constexpr int binary_format::mantissa_explicit_bits() { - return 52; -} +// big integer type. implements a small subset of big integer +// arithmetic, using simple algorithms since asymptotically +// faster algorithms are slower for a small number of limbs. +// all operations assume the big-integer is normalized. +struct bigint : pow5_tables<> { + // storage of the limbs, in little-endian order. + stackvec vec; -template <> constexpr int binary_format::minimum_exponent() { - return -1023; -} -template <> constexpr int binary_format::infinite_power() { - return 0x7FF; -} + SIMDJSON_FASTFLOAT_CONSTEXPR20 bigint() : vec() {} -template <> constexpr int binary_format::sign_index() { return 63; } + bigint(bigint const &) = delete; + bigint &operator=(bigint const &) = delete; + bigint(bigint &&) = delete; + bigint &operator=(bigint &&other) = delete; -bool is_integer(char c) noexcept { return (c >= '0' && c <= '9'); } + SIMDJSON_FASTFLOAT_CONSTEXPR20 bigint(uint64_t value) : vec() { +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + vec.push_unchecked(value); +#else + vec.push_unchecked(uint32_t(value)); + vec.push_unchecked(uint32_t(value >> 32)); +#endif + vec.normalize(); + } -// This should always succeed since it follows a call to parse_number. -decimal parse_decimal(const char *&p) noexcept { - decimal answer; - answer.num_digits = 0; - answer.decimal_point = 0; - answer.truncated = false; - answer.negative = (*p == '-'); - if ((*p == '-') || (*p == '+')) { - ++p; + // get the high 64 bits from the vector, and if bits were truncated. + // this is to get the significant digits for the float. + SIMDJSON_FASTFLOAT_CONSTEXPR20 uint64_t hi64(bool &truncated) const noexcept { +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + if (vec.len() == 0) { + return empty_hi64(truncated); + } else if (vec.len() == 1) { + return uint64_hi64(vec.rindex(0), truncated); + } else { + uint64_t result = uint64_hi64(vec.rindex(0), vec.rindex(1), truncated); + truncated |= vec.nonzero(2); + return result; + } +#else + if (vec.len() == 0) { + return empty_hi64(truncated); + } else if (vec.len() == 1) { + return uint32_hi64(vec.rindex(0), truncated); + } else if (vec.len() == 2) { + return uint32_hi64(vec.rindex(0), vec.rindex(1), truncated); + } else { + uint64_t result = + uint32_hi64(vec.rindex(0), vec.rindex(1), vec.rindex(2), truncated); + truncated |= vec.nonzero(3); + return result; + } +#endif } - while (*p == '0') { - ++p; + // compare two big integers, returning the large value. + // assumes both are normalized. if the return value is + // negative, other is larger, if the return value is + // positive, this is larger, otherwise they are equal. + // the limbs are stored in little-endian order, so we + // must compare the limbs in ever order. + SIMDJSON_FASTFLOAT_CONSTEXPR20 int compare(bigint const &other) const noexcept { + if (vec.len() > other.vec.len()) { + return 1; + } else if (vec.len() < other.vec.len()) { + return -1; + } else { + for (size_t index = vec.len(); index > 0; index--) { + limb xi = vec[index - 1]; + limb yi = other.vec[index - 1]; + if (xi > yi) { + return 1; + } else if (xi < yi) { + return -1; + } + } + return 0; + } } - while (is_integer(*p)) { - if (answer.num_digits < max_digits) { - answer.digits[answer.num_digits] = uint8_t(*p - '0'); + + // shift left each limb n bits, carrying over to the new limb + // returns true if we were able to shift all the digits. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool shl_bits(size_t n) noexcept { + // Internally, for each item, we shift left by n, and add the previous + // right shifted limb-bits. + // For example, we transform (for u8) shifted left 2, to: + // b10100100 b01000010 + // b10 b10010001 b00001000 + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(n != 0); + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(n < sizeof(limb) * 8); + + size_t shl = n; + size_t shr = limb_bits - shl; + limb prev = 0; + for (size_t index = 0; index < vec.len(); index++) { + limb xi = vec[index]; + vec[index] = (xi << shl) | (prev >> shr); + prev = xi; } - answer.num_digits++; - ++p; + + limb carry = prev >> shr; + if (carry != 0) { + return vec.try_push(carry); + } + return true; } - if (*p == '.') { - ++p; - const char *first_after_period = p; - // if we have not yet encountered a zero, we have to skip it as well - if (answer.num_digits == 0) { - // skip zeros - while (*p == '0') { - ++p; - } + + // move the limbs left by `n` limbs. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool shl_limbs(size_t n) noexcept { + SIMDJSON_FASTFLOAT_DEBUG_ASSERT(n != 0); + if (n + vec.len() > vec.capacity()) { + return false; + } else if (!vec.is_empty()) { + // move limbs + limb *dst = vec.data + n; + limb const *src = vec.data; + std::copy_backward(src, src + vec.len(), dst + vec.len()); + // fill in empty limbs + limb *first = vec.data; + limb *last = first + n; + ::std::fill(first, last, 0); + vec.set_len(n + vec.len()); + return true; + } else { + return true; } - while (is_integer(*p)) { - if (answer.num_digits < max_digits) { - answer.digits[answer.num_digits] = uint8_t(*p - '0'); - } - answer.num_digits++; - ++p; + } + + // move the limbs left by `n` bits. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool shl(size_t n) noexcept { + size_t rem = n % limb_bits; + size_t div = n / limb_bits; + if (rem != 0) { + SIMDJSON_FASTFLOAT_TRY(shl_bits(rem)); } - answer.decimal_point = int32_t(first_after_period - p); + if (div != 0) { + SIMDJSON_FASTFLOAT_TRY(shl_limbs(div)); + } + return true; } - if(answer.num_digits > 0) { - const char *preverse = p - 1; - int32_t trailing_zeros = 0; - while ((*preverse == '0') || (*preverse == '.')) { - if(*preverse == '0') { trailing_zeros++; }; - --preverse; + + // get the number of leading zeros in the bigint. + SIMDJSON_FASTFLOAT_CONSTEXPR20 int ctlz() const noexcept { + if (vec.is_empty()) { + return 0; + } else { +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + return leading_zeroes(vec.rindex(0)); +#else + // no use defining a specialized leading_zeroes for a 32-bit type. + uint64_t r0 = vec.rindex(0); + return leading_zeroes(r0 << 32); +#endif } - answer.decimal_point += int32_t(answer.num_digits); - answer.num_digits -= uint32_t(trailing_zeros); } - if(answer.num_digits > max_digits ) { - answer.num_digits = max_digits; - answer.truncated = true; + + // get the number of bits in the bigint. + SIMDJSON_FASTFLOAT_CONSTEXPR20 int bit_length() const noexcept { + int lz = ctlz(); + return int(limb_bits * vec.len()) - lz; } - if (('e' == *p) || ('E' == *p)) { - ++p; - bool neg_exp = false; - if ('-' == *p) { - neg_exp = true; - ++p; - } else if ('+' == *p) { - ++p; + + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool mul(limb y) noexcept { return small_mul(vec, y); } + + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool add(limb y) noexcept { return small_add(vec, y); } + + // multiply as if by 2 raised to a power. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool pow2(uint32_t exp) noexcept { return shl(exp); } + + // multiply as if by 5 raised to a power. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool pow5(uint32_t exp) noexcept { + // multiply by a power of 5 + size_t large_length = sizeof(large_power_of_5) / sizeof(limb); + limb_span large = limb_span(large_power_of_5, large_length); + while (exp >= large_step) { + SIMDJSON_FASTFLOAT_TRY(large_mul(vec, large)); + exp -= large_step; } - int32_t exp_number = 0; // exponential part - while (is_integer(*p)) { - uint8_t digit = uint8_t(*p - '0'); - if (exp_number < 0x10000) { - exp_number = 10 * exp_number + digit; - } - ++p; +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + uint32_t small_step = 27; + limb max_native = 7450580596923828125UL; +#else + uint32_t small_step = 13; + limb max_native = 1220703125U; +#endif + while (exp >= small_step) { + SIMDJSON_FASTFLOAT_TRY(small_mul(vec, max_native)); + exp -= small_step; + } + if (exp != 0) { + // Work around clang bug https://godbolt.org/z/zedh7rrhc + // This is similar to https://github.com/llvm/llvm-project/issues/47746, + // except the workaround described there don't work here + SIMDJSON_FASTFLOAT_TRY(small_mul(vec, limb((static_cast(small_power_of_5[0]), + small_power_of_5[exp])))); } - answer.decimal_point += (neg_exp ? -exp_number : exp_number); + + return true; } - return answer; + + // multiply as if by 10 raised to a power. + SIMDJSON_FASTFLOAT_CONSTEXPR20 bool pow10(uint32_t exp) noexcept { + SIMDJSON_FASTFLOAT_TRY(pow5(exp)); + return pow2(exp); + } +}; + +} // namespace simdjson_fast_float + +#endif + +#ifndef SIMDJSON_FASTFLOAT_DIGIT_COMPARISON_H +#define SIMDJSON_FASTFLOAT_DIGIT_COMPARISON_H + +#include +#include +#include + + +namespace simdjson_fast_float { + +// 1e0 to 1e19 +constexpr static uint64_t powers_of_ten_uint64[] = {1UL, + 10UL, + 100UL, + 1000UL, + 10000UL, + 100000UL, + 1000000UL, + 10000000UL, + 100000000UL, + 1000000000UL, + 10000000000UL, + 100000000000UL, + 1000000000000UL, + 10000000000000UL, + 100000000000000UL, + 1000000000000000UL, + 10000000000000000UL, + 100000000000000000UL, + 1000000000000000000UL, + 10000000000000000000UL}; + +// calculate the exponent, in scientific notation, of the number. +// this algorithm is not even close to optimized, but it has no practical +// effect on performance: in order to have a faster algorithm, we'd need +// to slow down performance for faster algorithms, and this is still fast. +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 int32_t +scientific_exponent(uint64_t mantissa, int32_t exponent) noexcept { + while (mantissa >= 10000) { + mantissa /= 10000; + exponent += 4; + } + while (mantissa >= 100) { + mantissa /= 100; + exponent += 2; + } + while (mantissa >= 10) { + mantissa /= 10; + exponent += 1; + } + return exponent; +} + +// this converts a native floating-point number to an extended-precision float. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +to_extended(T value) noexcept { + using equiv_uint = equiv_uint_t; + constexpr equiv_uint exponent_mask = binary_format::exponent_mask(); + constexpr equiv_uint mantissa_mask = binary_format::mantissa_mask(); + constexpr equiv_uint hidden_bit_mask = binary_format::hidden_bit_mask(); + + adjusted_mantissa am; + int32_t bias = binary_format::mantissa_explicit_bits() - + binary_format::minimum_exponent(); + equiv_uint bits; +#if SIMDJSON_FASTFLOAT_HAS_BIT_CAST + bits = std::bit_cast(value); +#else + ::memcpy(&bits, &value, sizeof(T)); +#endif + if ((bits & exponent_mask) == 0) { + // denormal + am.power2 = 1 - bias; + am.mantissa = bits & mantissa_mask; + } else { + // normal + am.power2 = int32_t((bits & exponent_mask) >> + binary_format::mantissa_explicit_bits()); + am.power2 -= bias; + am.mantissa = (bits & mantissa_mask) | hidden_bit_mask; + } + + return am; } -// This should always succeed since it follows a call to parse_number. -// Will not read at or beyond the "end" pointer. -decimal parse_decimal(const char *&p, const char * end) noexcept { - decimal answer; - answer.num_digits = 0; - answer.decimal_point = 0; - answer.truncated = false; - if(p == end) { return answer; } // should never happen - answer.negative = (*p == '-'); - if ((*p == '-') || (*p == '+')) { - ++p; +// get the extended precision value of the halfway point between b and b+u. +// we are given a native float that represents b, so we need to adjust it +// halfway between b and b+u. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +to_extended_halfway(T value) noexcept { + adjusted_mantissa am = to_extended(value); + am.mantissa <<= 1; + am.mantissa += 1; + am.power2 -= 1; + return am; +} + +// round an extended-precision float to the nearest machine float. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void round(adjusted_mantissa &am, + callback cb) noexcept { + int32_t mantissa_shift = 64 - binary_format::mantissa_explicit_bits() - 1; + if (-am.power2 >= mantissa_shift) { + // have a denormal float + int32_t shift = -am.power2 + 1; + cb(am, (shift < 64 ? shift : 64)); + // check for round-up: if rounding-nearest carried us to the hidden bit. + am.power2 = (am.mantissa < + (uint64_t(1) << binary_format::mantissa_explicit_bits())) + ? 0 + : 1; + return; } - while ((p != end) && (*p == '0')) { - ++p; + // have a normal float, use the default shift. + cb(am, mantissa_shift); + + // check for carry + if (am.mantissa >= + (uint64_t(2) << binary_format::mantissa_explicit_bits())) { + am.mantissa = (uint64_t(1) << binary_format::mantissa_explicit_bits()); + am.power2++; + } + + // check for infinite: we could have carried to an infinite power + am.mantissa &= ~(uint64_t(1) << binary_format::mantissa_explicit_bits()); + if (am.power2 >= binary_format::infinite_power()) { + am.power2 = binary_format::infinite_power(); + am.mantissa = 0; + } +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void +round_nearest_tie_even(adjusted_mantissa &am, int32_t shift, + callback cb) noexcept { + uint64_t const mask = (shift == 64) ? UINT64_MAX : (uint64_t(1) << shift) - 1; + uint64_t const halfway = (shift == 0) ? 0 : uint64_t(1) << (shift - 1); + uint64_t truncated_bits = am.mantissa & mask; + bool is_above = truncated_bits > halfway; + bool is_halfway = truncated_bits == halfway; + + // shift digits into position + if (shift == 64) { + am.mantissa = 0; + } else { + am.mantissa >>= shift; + } + am.power2 += shift; + + bool is_odd = (am.mantissa & 1) == 1; + am.mantissa += uint64_t(cb(is_odd, is_halfway, is_above)); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void +round_down(adjusted_mantissa &am, int32_t shift) noexcept { + if (shift == 64) { + am.mantissa = 0; + } else { + am.mantissa >>= shift; } - while ((p != end) && is_integer(*p)) { - if (answer.num_digits < max_digits) { - answer.digits[answer.num_digits] = uint8_t(*p - '0'); + am.power2 += shift; +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +skip_zeros(UC const *&first, UC const *last) noexcept { + uint64_t val; + while (!cpp20_and_in_constexpr() && + std::distance(first, last) >= int_cmp_len()) { + ::memcpy(&val, first, sizeof(uint64_t)); + if (val != int_cmp_zeros()) { + break; } - answer.num_digits++; - ++p; + first += int_cmp_len(); } - if ((p != end) && (*p == '.')) { - ++p; - if(p == end) { return answer; } // should never happen - const char *first_after_period = p; - // if we have not yet encountered a zero, we have to skip it as well - if (answer.num_digits == 0) { - // skip zeros - while (*p == '0') { - ++p; - } + while (first != last) { + if (*first != UC('0')) { + break; } - while ((p != end) && is_integer(*p)) { - if (answer.num_digits < max_digits) { - answer.digits[answer.num_digits] = uint8_t(*p - '0'); - } - answer.num_digits++; - ++p; + first++; + } +} + +// determine if any non-zero digits were truncated. +// all characters must be valid digits. +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +is_truncated(UC const *first, UC const *last) noexcept { + // do 8-bit optimizations, can just compare to 8 literal 0s. + uint64_t val; + while (!cpp20_and_in_constexpr() && + std::distance(first, last) >= int_cmp_len()) { + ::memcpy(&val, first, sizeof(uint64_t)); + if (val != int_cmp_zeros()) { + return true; } - answer.decimal_point = int32_t(first_after_period - p); + first += int_cmp_len(); } - if(answer.num_digits > 0) { - const char *preverse = p - 1; - int32_t trailing_zeros = 0; - while ((*preverse == '0') || (*preverse == '.')) { - if(*preverse == '0') { trailing_zeros++; }; - --preverse; + while (first != last) { + if (*first != UC('0')) { + return true; } - answer.decimal_point += int32_t(answer.num_digits); - answer.num_digits -= uint32_t(trailing_zeros); + ++first; } - if(answer.num_digits > max_digits ) { - answer.num_digits = max_digits; - answer.truncated = true; + return false; +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +is_truncated(span s) noexcept { + return is_truncated(s.ptr, s.ptr + s.len()); +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +parse_eight_digits(UC const *&p, limb &value, size_t &counter, + size_t &count) noexcept { + value = value * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + counter += 8; + count += 8; +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR14 void +parse_one_digit(UC const *&p, limb &value, size_t &counter, + size_t &count) noexcept { + value = value * 10 + limb(*p - UC('0')); + p++; + counter++; + count++; +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +add_native(bigint &big, limb power, limb value) noexcept { + big.mul(power); + big.add(value); +} + +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +round_up_bigint(bigint &big, size_t &count) noexcept { + // need to round-up the digits, but need to avoid rounding + // ....9999 to ...10000, which could cause a false halfway point. + add_native(big, 10, 1); + count++; +} + +// parse the significant digits into a big integer +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 void +parse_mantissa(bigint &result, parsed_number_string_t &num, + size_t max_digits, size_t &digits) noexcept { + // try to minimize the number of big integer and scalar multiplication. + // therefore, try to parse 8 digits at a time, and multiply by the largest + // scalar value (9 or 19 digits) for each step. + size_t counter = 0; + digits = 0; + limb value = 0; +#ifdef SIMDJSON_FASTFLOAT_64BIT_LIMB + size_t step = 19; +#else + size_t step = 9; +#endif + + // process all integer digits. + UC const *p = num.integer.ptr; + UC const *pend = p + num.integer.len(); + skip_zeros(p, pend); + // process all digits, in increments of step per loop + while (p != pend) { + while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && + (max_digits - digits >= 8)) { + parse_eight_digits(p, value, counter, digits); + } + while (counter < step && p != pend && digits < max_digits) { + parse_one_digit(p, value, counter, digits); + } + if (digits == max_digits) { + // add the temporary value, then check if we've truncated any digits + add_native(result, limb(powers_of_ten_uint64[counter]), value); + bool truncated = is_truncated(p, pend); + if (num.fraction.ptr != nullptr) { + truncated |= is_truncated(num.fraction); + } + if (truncated) { + round_up_bigint(result, digits); + } + return; + } else { + add_native(result, limb(powers_of_ten_uint64[counter]), value); + counter = 0; + value = 0; + } } - if ((p != end) && (('e' == *p) || ('E' == *p))) { - ++p; - if(p == end) { return answer; } // should never happen - bool neg_exp = false; - if ('-' == *p) { - neg_exp = true; - ++p; - } else if ('+' == *p) { - ++p; + + // add our fraction digits, if they're available. + if (num.fraction.ptr != nullptr) { + p = num.fraction.ptr; + pend = p + num.fraction.len(); + if (digits == 0) { + skip_zeros(p, pend); } - int32_t exp_number = 0; // exponential part - while ((p != end) && is_integer(*p)) { - uint8_t digit = uint8_t(*p - '0'); - if (exp_number < 0x10000) { - exp_number = 10 * exp_number + digit; + // process all digits, in increments of step per loop + while (p != pend) { + while ((std::distance(p, pend) >= 8) && (step - counter >= 8) && + (max_digits - digits >= 8)) { + parse_eight_digits(p, value, counter, digits); + } + while (counter < step && p != pend && digits < max_digits) { + parse_one_digit(p, value, counter, digits); + } + if (digits == max_digits) { + // add the temporary value, then check if we've truncated any digits + add_native(result, limb(powers_of_ten_uint64[counter]), value); + bool truncated = is_truncated(p, pend); + if (truncated) { + round_up_bigint(result, digits); + } + return; + } else { + add_native(result, limb(powers_of_ten_uint64[counter]), value); + counter = 0; + value = 0; } - ++p; } - answer.decimal_point += (neg_exp ? -exp_number : exp_number); } + + if (counter != 0) { + add_native(result, limb(powers_of_ten_uint64[counter]), value); + } +} + +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +positive_digit_comp(bigint &bigmant, int32_t exponent) noexcept { + SIMDJSON_FASTFLOAT_ASSERT(bigmant.pow10(uint32_t(exponent))); + adjusted_mantissa answer; + bool truncated; + answer.mantissa = bigmant.hi64(truncated); + int bias = binary_format::mantissa_explicit_bits() - + binary_format::minimum_exponent(); + answer.power2 = bigmant.bit_length() - 64 + bias; + + round(answer, [truncated](adjusted_mantissa &a, int32_t shift) { + round_nearest_tie_even( + a, shift, + [truncated](bool is_odd, bool is_halfway, bool is_above) -> bool { + return is_above || (is_halfway && truncated) || + (is_odd && is_halfway); + }); + }); + return answer; } -namespace { +// the scaling here is quite simple: we have, for the real digits `m * 10^e`, +// and for the theoretical digits `n * 2^f`. Since `e` is always negative, +// to scale them identically, we do `n * 2^f * 5^-f`, so we now have `m * 2^e`. +// we then need to scale by `2^(f- e)`, and then the two significant digits +// are of the same magnitude. +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa negative_digit_comp( + bigint &bigmant, adjusted_mantissa am, int32_t exponent) noexcept { + bigint &real_digits = bigmant; + int32_t real_exp = exponent; + + // get the value of `b`, rounded down, and get a bigint representation of b+h + adjusted_mantissa am_b = am; + // gcc7 buf: use a lambda to remove the noexcept qualifier bug with + // -Wnoexcept-type. + round(am_b, + [](adjusted_mantissa &a, int32_t shift) { round_down(a, shift); }); + T b; + to_float(false, am_b, b); + adjusted_mantissa theor = to_extended_halfway(b); + bigint theor_digits(theor.mantissa); + int32_t theor_exp = theor.power2; + + // scale real digits and theor digits to be same power. + int32_t pow2_exp = theor_exp - real_exp; + uint32_t pow5_exp = uint32_t(-real_exp); + if (pow5_exp != 0) { + SIMDJSON_FASTFLOAT_ASSERT(theor_digits.pow5(pow5_exp)); + } + if (pow2_exp > 0) { + SIMDJSON_FASTFLOAT_ASSERT(theor_digits.pow2(uint32_t(pow2_exp))); + } else if (pow2_exp < 0) { + SIMDJSON_FASTFLOAT_ASSERT(real_digits.pow2(uint32_t(-pow2_exp))); + } + + // compare digits, and use it to direct rounding + int ord = real_digits.compare(theor_digits); + adjusted_mantissa answer = am; + round(answer, [ord](adjusted_mantissa &a, int32_t shift) { + round_nearest_tie_even( + a, shift, [ord](bool is_odd, bool _, bool __) -> bool { + static_cast(_); // not needed, since we've done our comparison + static_cast(__); // not needed, since we've done our comparison + if (ord > 0) { + return true; + } else if (ord < 0) { + return false; + } else { + return is_odd; + } + }); + }); + + return answer; +} -// remove all final zeroes -inline void trim(decimal &h) { - while ((h.num_digits > 0) && (h.digits[h.num_digits - 1] == 0)) { - h.num_digits--; +// parse the significant digits as a big integer to unambiguously round +// the significant digits. here, we are trying to determine how to round +// an extended float representation close to `b+h`, halfway between `b` +// (the float rounded-down) and `b+u`, the next positive float. this +// algorithm is always correct, and uses one of two approaches. when +// the exponent is positive relative to the significant digits (such as +// 1234), we create a big-integer representation, get the high 64-bits, +// determine if any lower bits are truncated, and use that to direct +// rounding. in case of a negative exponent relative to the significant +// digits (such as 1.2345), we create a theoretical representation of +// `b` as a big-integer type, scaled to the same binary exponent as +// the actual digits. we then compare the big integer representations +// of both, and use that to direct rounding. +template +inline SIMDJSON_FASTFLOAT_CONSTEXPR20 adjusted_mantissa +digit_comp(parsed_number_string_t &num, adjusted_mantissa am) noexcept { + // remove the invalid exponent bias + am.power2 -= invalid_am_bias; + + int32_t sci_exp = + scientific_exponent(num.mantissa, static_cast(num.exponent)); + size_t max_digits = binary_format::max_digits(); + size_t digits = 0; + bigint bigmant; + parse_mantissa(bigmant, num, max_digits, digits); + // can't underflow, since digits is at most max_digits. + int32_t exponent = sci_exp + 1 - int32_t(digits); + if (exponent >= 0) { + return positive_digit_comp(bigmant, exponent); + } else { + return negative_digit_comp(bigmant, am, exponent); } } -uint32_t number_of_digits_decimal_left_shift(decimal &h, uint32_t shift) { - shift &= 63; - const static uint16_t number_of_digits_decimal_left_shift_table[65] = { - 0x0000, 0x0800, 0x0801, 0x0803, 0x1006, 0x1009, 0x100D, 0x1812, 0x1817, - 0x181D, 0x2024, 0x202B, 0x2033, 0x203C, 0x2846, 0x2850, 0x285B, 0x3067, - 0x3073, 0x3080, 0x388E, 0x389C, 0x38AB, 0x38BB, 0x40CC, 0x40DD, 0x40EF, - 0x4902, 0x4915, 0x4929, 0x513E, 0x5153, 0x5169, 0x5180, 0x5998, 0x59B0, - 0x59C9, 0x61E3, 0x61FD, 0x6218, 0x6A34, 0x6A50, 0x6A6D, 0x6A8B, 0x72AA, - 0x72C9, 0x72E9, 0x7B0A, 0x7B2B, 0x7B4D, 0x8370, 0x8393, 0x83B7, 0x83DC, - 0x8C02, 0x8C28, 0x8C4F, 0x9477, 0x949F, 0x94C8, 0x9CF2, 0x051C, 0x051C, - 0x051C, 0x051C, - }; - uint32_t x_a = number_of_digits_decimal_left_shift_table[shift]; - uint32_t x_b = number_of_digits_decimal_left_shift_table[shift + 1]; - uint32_t num_new_digits = x_a >> 11; - uint32_t pow5_a = 0x7FF & x_a; - uint32_t pow5_b = 0x7FF & x_b; - const static uint8_t - number_of_digits_decimal_left_shift_table_powers_of_5[0x051C] = { - 5, 2, 5, 1, 2, 5, 6, 2, 5, 3, 1, 2, 5, 1, 5, 6, 2, 5, 7, 8, 1, 2, 5, - 3, 9, 0, 6, 2, 5, 1, 9, 5, 3, 1, 2, 5, 9, 7, 6, 5, 6, 2, 5, 4, 8, 8, - 2, 8, 1, 2, 5, 2, 4, 4, 1, 4, 0, 6, 2, 5, 1, 2, 2, 0, 7, 0, 3, 1, 2, - 5, 6, 1, 0, 3, 5, 1, 5, 6, 2, 5, 3, 0, 5, 1, 7, 5, 7, 8, 1, 2, 5, 1, - 5, 2, 5, 8, 7, 8, 9, 0, 6, 2, 5, 7, 6, 2, 9, 3, 9, 4, 5, 3, 1, 2, 5, - 3, 8, 1, 4, 6, 9, 7, 2, 6, 5, 6, 2, 5, 1, 9, 0, 7, 3, 4, 8, 6, 3, 2, - 8, 1, 2, 5, 9, 5, 3, 6, 7, 4, 3, 1, 6, 4, 0, 6, 2, 5, 4, 7, 6, 8, 3, - 7, 1, 5, 8, 2, 0, 3, 1, 2, 5, 2, 3, 8, 4, 1, 8, 5, 7, 9, 1, 0, 1, 5, - 6, 2, 5, 1, 1, 9, 2, 0, 9, 2, 8, 9, 5, 5, 0, 7, 8, 1, 2, 5, 5, 9, 6, - 0, 4, 6, 4, 4, 7, 7, 5, 3, 9, 0, 6, 2, 5, 2, 9, 8, 0, 2, 3, 2, 2, 3, - 8, 7, 6, 9, 5, 3, 1, 2, 5, 1, 4, 9, 0, 1, 1, 6, 1, 1, 9, 3, 8, 4, 7, - 6, 5, 6, 2, 5, 7, 4, 5, 0, 5, 8, 0, 5, 9, 6, 9, 2, 3, 8, 2, 8, 1, 2, - 5, 3, 7, 2, 5, 2, 9, 0, 2, 9, 8, 4, 6, 1, 9, 1, 4, 0, 6, 2, 5, 1, 8, - 6, 2, 6, 4, 5, 1, 4, 9, 2, 3, 0, 9, 5, 7, 0, 3, 1, 2, 5, 9, 3, 1, 3, - 2, 2, 5, 7, 4, 6, 1, 5, 4, 7, 8, 5, 1, 5, 6, 2, 5, 4, 6, 5, 6, 6, 1, - 2, 8, 7, 3, 0, 7, 7, 3, 9, 2, 5, 7, 8, 1, 2, 5, 2, 3, 2, 8, 3, 0, 6, - 4, 3, 6, 5, 3, 8, 6, 9, 6, 2, 8, 9, 0, 6, 2, 5, 1, 1, 6, 4, 1, 5, 3, - 2, 1, 8, 2, 6, 9, 3, 4, 8, 1, 4, 4, 5, 3, 1, 2, 5, 5, 8, 2, 0, 7, 6, - 6, 0, 9, 1, 3, 4, 6, 7, 4, 0, 7, 2, 2, 6, 5, 6, 2, 5, 2, 9, 1, 0, 3, - 8, 3, 0, 4, 5, 6, 7, 3, 3, 7, 0, 3, 6, 1, 3, 2, 8, 1, 2, 5, 1, 4, 5, - 5, 1, 9, 1, 5, 2, 2, 8, 3, 6, 6, 8, 5, 1, 8, 0, 6, 6, 4, 0, 6, 2, 5, - 7, 2, 7, 5, 9, 5, 7, 6, 1, 4, 1, 8, 3, 4, 2, 5, 9, 0, 3, 3, 2, 0, 3, - 1, 2, 5, 3, 6, 3, 7, 9, 7, 8, 8, 0, 7, 0, 9, 1, 7, 1, 2, 9, 5, 1, 6, - 6, 0, 1, 5, 6, 2, 5, 1, 8, 1, 8, 9, 8, 9, 4, 0, 3, 5, 4, 5, 8, 5, 6, - 4, 7, 5, 8, 3, 0, 0, 7, 8, 1, 2, 5, 9, 0, 9, 4, 9, 4, 7, 0, 1, 7, 7, - 2, 9, 2, 8, 2, 3, 7, 9, 1, 5, 0, 3, 9, 0, 6, 2, 5, 4, 5, 4, 7, 4, 7, - 3, 5, 0, 8, 8, 6, 4, 6, 4, 1, 1, 8, 9, 5, 7, 5, 1, 9, 5, 3, 1, 2, 5, - 2, 2, 7, 3, 7, 3, 6, 7, 5, 4, 4, 3, 2, 3, 2, 0, 5, 9, 4, 7, 8, 7, 5, - 9, 7, 6, 5, 6, 2, 5, 1, 1, 3, 6, 8, 6, 8, 3, 7, 7, 2, 1, 6, 1, 6, 0, - 2, 9, 7, 3, 9, 3, 7, 9, 8, 8, 2, 8, 1, 2, 5, 5, 6, 8, 4, 3, 4, 1, 8, - 8, 6, 0, 8, 0, 8, 0, 1, 4, 8, 6, 9, 6, 8, 9, 9, 4, 1, 4, 0, 6, 2, 5, - 2, 8, 4, 2, 1, 7, 0, 9, 4, 3, 0, 4, 0, 4, 0, 0, 7, 4, 3, 4, 8, 4, 4, - 9, 7, 0, 7, 0, 3, 1, 2, 5, 1, 4, 2, 1, 0, 8, 5, 4, 7, 1, 5, 2, 0, 2, - 0, 0, 3, 7, 1, 7, 4, 2, 2, 4, 8, 5, 3, 5, 1, 5, 6, 2, 5, 7, 1, 0, 5, - 4, 2, 7, 3, 5, 7, 6, 0, 1, 0, 0, 1, 8, 5, 8, 7, 1, 1, 2, 4, 2, 6, 7, - 5, 7, 8, 1, 2, 5, 3, 5, 5, 2, 7, 1, 3, 6, 7, 8, 8, 0, 0, 5, 0, 0, 9, - 2, 9, 3, 5, 5, 6, 2, 1, 3, 3, 7, 8, 9, 0, 6, 2, 5, 1, 7, 7, 6, 3, 5, - 6, 8, 3, 9, 4, 0, 0, 2, 5, 0, 4, 6, 4, 6, 7, 7, 8, 1, 0, 6, 6, 8, 9, - 4, 5, 3, 1, 2, 5, 8, 8, 8, 1, 7, 8, 4, 1, 9, 7, 0, 0, 1, 2, 5, 2, 3, - 2, 3, 3, 8, 9, 0, 5, 3, 3, 4, 4, 7, 2, 6, 5, 6, 2, 5, 4, 4, 4, 0, 8, - 9, 2, 0, 9, 8, 5, 0, 0, 6, 2, 6, 1, 6, 1, 6, 9, 4, 5, 2, 6, 6, 7, 2, - 3, 6, 3, 2, 8, 1, 2, 5, 2, 2, 2, 0, 4, 4, 6, 0, 4, 9, 2, 5, 0, 3, 1, - 3, 0, 8, 0, 8, 4, 7, 2, 6, 3, 3, 3, 6, 1, 8, 1, 6, 4, 0, 6, 2, 5, 1, - 1, 1, 0, 2, 2, 3, 0, 2, 4, 6, 2, 5, 1, 5, 6, 5, 4, 0, 4, 2, 3, 6, 3, - 1, 6, 6, 8, 0, 9, 0, 8, 2, 0, 3, 1, 2, 5, 5, 5, 5, 1, 1, 1, 5, 1, 2, - 3, 1, 2, 5, 7, 8, 2, 7, 0, 2, 1, 1, 8, 1, 5, 8, 3, 4, 0, 4, 5, 4, 1, - 0, 1, 5, 6, 2, 5, 2, 7, 7, 5, 5, 5, 7, 5, 6, 1, 5, 6, 2, 8, 9, 1, 3, - 5, 1, 0, 5, 9, 0, 7, 9, 1, 7, 0, 2, 2, 7, 0, 5, 0, 7, 8, 1, 2, 5, 1, - 3, 8, 7, 7, 7, 8, 7, 8, 0, 7, 8, 1, 4, 4, 5, 6, 7, 5, 5, 2, 9, 5, 3, - 9, 5, 8, 5, 1, 1, 3, 5, 2, 5, 3, 9, 0, 6, 2, 5, 6, 9, 3, 8, 8, 9, 3, - 9, 0, 3, 9, 0, 7, 2, 2, 8, 3, 7, 7, 6, 4, 7, 6, 9, 7, 9, 2, 5, 5, 6, - 7, 6, 2, 6, 9, 5, 3, 1, 2, 5, 3, 4, 6, 9, 4, 4, 6, 9, 5, 1, 9, 5, 3, - 6, 1, 4, 1, 8, 8, 8, 2, 3, 8, 4, 8, 9, 6, 2, 7, 8, 3, 8, 1, 3, 4, 7, - 6, 5, 6, 2, 5, 1, 7, 3, 4, 7, 2, 3, 4, 7, 5, 9, 7, 6, 8, 0, 7, 0, 9, - 4, 4, 1, 1, 9, 2, 4, 4, 8, 1, 3, 9, 1, 9, 0, 6, 7, 3, 8, 2, 8, 1, 2, - 5, 8, 6, 7, 3, 6, 1, 7, 3, 7, 9, 8, 8, 4, 0, 3, 5, 4, 7, 2, 0, 5, 9, - 6, 2, 2, 4, 0, 6, 9, 5, 9, 5, 3, 3, 6, 9, 1, 4, 0, 6, 2, 5, - }; - const uint8_t *pow5 = - &number_of_digits_decimal_left_shift_table_powers_of_5[pow5_a]; - uint32_t i = 0; - uint32_t n = pow5_b - pow5_a; - for (; i < n; i++) { - if (i >= h.num_digits) { - return num_new_digits - 1; - } else if (h.digits[i] == pow5[i]) { - continue; - } else if (h.digits[i] < pow5[i]) { - return num_new_digits - 1; - } else { - return num_new_digits; +} // namespace simdjson_fast_float + +#endif + +#ifndef SIMDJSON_FASTFLOAT_PARSE_NUMBER_H +#define SIMDJSON_FASTFLOAT_PARSE_NUMBER_H + + +#include +#include +#include +#include + +namespace simdjson_fast_float { + +namespace detail { +/** + * Special case +inf, -inf, nan, infinity, -infinity. + * The case comparisons could be made much faster given that we know that the + * strings a null-free and fixed. + **/ +template +from_chars_result_t + SIMDJSON_FASTFLOAT_CONSTEXPR14 parse_infnan(UC const *first, UC const *last, + T &value, chars_format fmt) noexcept { + from_chars_result_t answer{}; + answer.ptr = first; + answer.ec = std::errc(); // be optimistic + // assume first < last, so dereference without checks; + bool const minusSign = (*first == UC('-')); + // C++17 20.19.3.(7.1) explicitly forbids '+' sign here + if ((*first == UC('-')) || + (uint64_t(fmt & chars_format::allow_leading_plus) && + (*first == UC('+')))) { + ++first; + } + if (last - first >= 3) { + if (simdjson_fastfloat_strncasecmp3(first, str_const_nan())) { + answer.ptr = (first += 3); + value = minusSign ? -std::numeric_limits::quiet_NaN() + : std::numeric_limits::quiet_NaN(); + // Check for possible nan(n-char-seq-opt), C++17 20.19.3.7, + // C11 7.20.1.3.3. At least MSVC produces nan(ind) and nan(snan). + if (first != last && *first == UC('(')) { + for (UC const *ptr = first + 1; ptr != last; ++ptr) { + if (*ptr == UC(')')) { + answer.ptr = ptr + 1; // valid nan(n-char-seq-opt) + break; + } else if (!((UC('a') <= *ptr && *ptr <= UC('z')) || + (UC('A') <= *ptr && *ptr <= UC('Z')) || + (UC('0') <= *ptr && *ptr <= UC('9')) || *ptr == UC('_'))) + break; // forbidden char, not nan(n-char-seq-opt) + } + } + return answer; + } + if (simdjson_fastfloat_strncasecmp3(first, str_const_inf())) { + if ((last - first >= 8) && + simdjson_fastfloat_strncasecmp5(first + 3, str_const_inf() + 3)) { + answer.ptr = first + 8; + } else { + answer.ptr = first + 3; + } + value = minusSign ? -std::numeric_limits::infinity() + : std::numeric_limits::infinity(); + return answer; } } - return num_new_digits; + answer.ec = std::errc::invalid_argument; + return answer; +} + +/** + * Returns true if the floating-pointing rounding mode is to 'nearest'. + * It is the default on most system. This function is meant to be inexpensive. + * Credit : @mwalcott3 + */ +simdjson_fastfloat_really_inline bool rounds_to_nearest() noexcept { + // https://lemire.me/blog/2020/06/26/gcc-not-nearest/ +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + return false; +#endif + // See + // A fast function to check your floating-point rounding mode + // https://lemire.me/blog/2022/11/16/a-fast-function-to-check-your-floating-point-rounding-mode/ + // + // This function is meant to be equivalent to : + // prior: #include + // return fegetround() == FE_TONEAREST; + // However, it is expected to be much faster than the fegetround() + // function call. + // + // The volatile keyword prevents the compiler from computing the function + // at compile-time. + // There might be other ways to prevent compile-time optimizations (e.g., + // asm). The value does not need to be std::numeric_limits::min(), any + // small value so that 1 + x should round to 1 would do (after accounting for + // excess precision, as in 387 instructions). + static float volatile fmin = (std::numeric_limits::min)(); + float fmini = fmin; // we copy it so that it gets loaded at most once. +// +// Explanation: +// Only when fegetround() == FE_TONEAREST do we have that +// fmin + 1.0f == 1.0f - fmin. +// +// FE_UPWARD: +// fmin + 1.0f > 1 +// 1.0f - fmin == 1 +// +// FE_DOWNWARD or FE_TOWARDZERO: +// fmin + 1.0f == 1 +// 1.0f - fmin < 1 +// +// Note: This may fail to be accurate if fast-math has been +// enabled, as rounding conventions may not apply. +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(push) +// todo: is there a VS warning? +// see +// https://stackoverflow.com/questions/46079446/is-there-a-warning-for-floating-point-equality-checking-in-visual-studio-2013 +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wfloat-equal" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + return (fmini + 1.0f == 1.0f - fmini); +#ifdef SIMDJSON_FASTFLOAT_VISUAL_STUDIO +#pragma warning(pop) +#elif defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif } -} // end of anonymous namespace +} // namespace detail -uint64_t round(decimal &h) { - if ((h.num_digits == 0) || (h.decimal_point < 0)) { - return 0; - } else if (h.decimal_point > 18) { - return UINT64_MAX; - } - // at this point, we know that h.decimal_point >= 0 - uint32_t dp = uint32_t(h.decimal_point); - uint64_t n = 0; - for (uint32_t i = 0; i < dp; i++) { - n = (10 * n) + ((i < h.num_digits) ? h.digits[i] : 0); +template struct from_chars_caller { + template + SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t + call(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + return from_chars_advanced(first, last, value, options); } - bool round_up = false; - if (dp < h.num_digits) { - round_up = h.digits[dp] >= 5; // normally, we round up - // but we may need to round to even! - if ((h.digits[dp] == 5) && (dp + 1 == h.num_digits)) { - round_up = h.truncated || ((dp > 0) && (1 & h.digits[dp - 1])); - } +}; + +#ifdef __STDCPP_FLOAT32_T__ +template <> struct from_chars_caller { + template + SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t + call(UC const *first, UC const *last, std::float32_t &value, + parse_options_t options) noexcept { + // if std::float32_t is defined, and we are in C++23 mode; macro set for + // float32; set value to float due to equivalence between float and + // float32_t + float val = 0.0f; + auto ret = from_chars_advanced(first, last, val, options); + value = val; + return ret; } - if (round_up) { - n++; +}; +#endif + +#ifdef __STDCPP_FLOAT64_T__ +template <> struct from_chars_caller { + template + SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t + call(UC const *first, UC const *last, std::float64_t &value, + parse_options_t options) noexcept { + // if std::float64_t is defined, and we are in C++23 mode; macro set for + // float64; set value as double due to equivalence between double and + // float64_t + double val = 0.0; + auto ret = from_chars_advanced(first, last, val, options); + value = val; + return ret; } - return n; +}; +#endif + +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars(UC const *first, UC const *last, T &value, + chars_format fmt /*= chars_format::general*/) noexcept { + return from_chars_caller::call(first, last, value, + parse_options_t(fmt)); } -// computes h * 2^-shift -void decimal_left_shift(decimal &h, uint32_t shift) { - if (h.num_digits == 0) { - return; - } - uint32_t num_new_digits = number_of_digits_decimal_left_shift(h, shift); - int32_t read_index = int32_t(h.num_digits - 1); - uint32_t write_index = h.num_digits - 1 + num_new_digits; - uint64_t n = 0; - - while (read_index >= 0) { - n += uint64_t(h.digits[read_index]) << shift; - uint64_t quotient = n / 10; - uint64_t remainder = n - (10 * quotient); - if (write_index < max_digits) { - h.digits[write_index] = uint8_t(remainder); - } else if (remainder > 0) { - h.truncated = true; - } - n = quotient; - write_index--; - read_index--; - } - while (n > 0) { - uint64_t quotient = n / 10; - uint64_t remainder = n - (10 * quotient); - if (write_index < max_digits) { - h.digits[write_index] = uint8_t(remainder); - } else if (remainder > 0) { - h.truncated = true; - } - n = quotient; - write_index--; - } - h.num_digits += num_new_digits; - if (h.num_digits > max_digits) { - h.num_digits = max_digits; - } - h.decimal_point += int32_t(num_new_digits); - trim(h); -} - -// computes h * 2^shift -void decimal_right_shift(decimal &h, uint32_t shift) { - uint32_t read_index = 0; - uint32_t write_index = 0; - - uint64_t n = 0; - - while ((n >> shift) == 0) { - if (read_index < h.num_digits) { - n = (10 * n) + h.digits[read_index++]; - } else if (n == 0) { - return; +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 bool +clinger_fast_path_impl(uint64_t mantissa, int64_t exponent, bool is_negative, + T &value) noexcept { + // The implementation of the Clinger's fast path is convoluted because + // we want round-to-nearest in all cases, irrespective of the rounding mode + // selected on the thread. + // We proceed optimistically, assuming that detail::rounds_to_nearest() + // returns true. + if (binary_format::min_exponent_fast_path() <= exponent && + exponent <= binary_format::max_exponent_fast_path() && + mantissa <= binary_format::max_mantissa_fast_path()) { + // The mantissa bound above is a necessary condition for BOTH branches + // below: the rounding-mode-dependent branch checks the tighter + // max_mantissa_fast_path(exponent) <= max_mantissa_fast_path(). Testing + // it before detail::rounds_to_nearest() spares long-mantissa inputs + // (which can never take the fast path) the volatile-float probe. + // + // Unfortunately, the conventional Clinger's fast path is only possible + // when the system rounds to the nearest float. + // + // We expect the next branch to almost always be selected. + // We could check it first (before the previous branch), but + // there might be performance advantages at having the check + // be last. + if (!cpp20_and_in_constexpr() && detail::rounds_to_nearest()) { + // We have that fegetround() == FE_TONEAREST. + // Next is Clinger's fast path. + value = T(mantissa); + if (exponent < 0) { + value = value / binary_format::exact_power_of_ten(-exponent); + } else { + value = value * binary_format::exact_power_of_ten(exponent); + } + if (is_negative) { + value = -value; + } + return true; } else { - while ((n >> shift) == 0) { - n = 10 * n; - read_index++; + // We do not have that fegetround() == FE_TONEAREST. + // Next is a modified Clinger's fast path, inspired by Jakub Jelinek's + // proposal + if (exponent >= 0 && + mantissa <= binary_format::max_mantissa_fast_path(exponent)) { +#if defined(__clang__) || defined(SIMDJSON_FASTFLOAT_32BIT) + // Clang may map 0 to -0.0 when fegetround() == FE_DOWNWARD + if (mantissa == 0) { + value = is_negative ? T(-0.) : T(0.); + return true; + } +#endif + value = T(mantissa) * binary_format::exact_power_of_ten(exponent); + if (is_negative) { + value = -value; + } + return true; } - break; } } - h.decimal_point -= int32_t(read_index - 1); - if (h.decimal_point < -decimal_point_range) { // it is zero - h.num_digits = 0; - h.decimal_point = 0; - h.negative = false; - h.truncated = false; - return; + return false; +} + +/** + * This function overload takes parsed_number_string_t structure that is created + * and populated either by from_chars_advanced function taking chars range and + * parsing options or other parsing custom function implemented by user. + */ +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_advanced(parsed_number_string_t &pns, T &value) noexcept { + static_assert(is_supported_float_type::value, + "only some floating-point types are supported"); + static_assert(is_supported_char_type::value, + "only char, wchar_t, char16_t and char32_t are supported"); + + from_chars_result_t answer; + + answer.ec = std::errc(); // be optimistic + answer.ptr = pns.lastmatch; + + if (!pns.too_many_digits && + clinger_fast_path_impl(pns.mantissa, pns.exponent, pns.negative, value)) + return answer; + + adjusted_mantissa am = + compute_float>(pns.exponent, pns.mantissa); + if (pns.too_many_digits && am.power2 >= 0) { + if (am != compute_float>(pns.exponent, pns.mantissa + 1)) { + am = compute_error>(pns.exponent, pns.mantissa); + } } - uint64_t mask = (uint64_t(1) << shift) - 1; - while (read_index < h.num_digits) { - uint8_t new_digit = uint8_t(n >> shift); - n = (10 * (n & mask)) + h.digits[read_index++]; - h.digits[write_index++] = new_digit; + // If we called compute_float>(pns.exponent, pns.mantissa) + // and we have an invalid power (am.power2 < 0), then we need to go the long + // way around again. This is very uncommon. + if (am.power2 < 0) { + am = digit_comp(pns, am); } - while (n > 0) { - uint8_t new_digit = uint8_t(n >> shift); - n = 10 * (n & mask); - if (write_index < max_digits) { - h.digits[write_index++] = new_digit; - } else if (new_digit > 0) { - h.truncated = true; - } + to_float(pns.negative, am, value); + // Test for over/underflow. + if ((pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) || + am.power2 == binary_format::infinite_power()) { + answer.ec = std::errc::result_out_of_range; } - h.num_digits = write_index; - trim(h); + return answer; } -template adjusted_mantissa compute_float(decimal &d) { - adjusted_mantissa answer; - if (d.num_digits == 0) { - // should be zero - answer.power2 = 0; - answer.mantissa = 0; - return answer; - } - // At this point, going further, we can assume that d.num_digits > 0. - // We want to guard against excessive decimal point values because - // they can result in long running times. Indeed, we do - // shifts by at most 60 bits. We have that log(10**400)/log(2**60) ~= 22 - // which is fine, but log(10**299995)/log(2**60) ~= 16609 which is not - // fine (runs for a long time). - // - if(d.decimal_point < -324) { - // We have something smaller than 1e-324 which is always zero - // in binary64 and binary32. - // It should be zero. - answer.power2 = 0; - answer.mantissa = 0; - return answer; - } else if(d.decimal_point >= 310) { - // We have something at least as large as 0.1e310 which is - // always infinite. - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; +// Slow path: re-parse materializing the integer/fraction spans the hot no-span +// parse skipped, then run the full algorithm. The two callers reach it only +// through a simdjson_fastfloat_unlikely branch, so the optimizer keeps this re-parse off +// the hot path on its own (no function-level noinline needed). +// from_chars_advanced already handles both the too_many_digits disambiguation +// and the am.power2<0 digit_comp recompute, so both slow branches collapse to +// one helper call. +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +parse_number_slow_path(UC const *first, UC const *last, T &value, + parse_options_t options, bool bjf) noexcept { + parsed_number_string_t pns = + bjf ? parse_number_string(first, last, options, true) + : parse_number_string(first, last, options, true); + return from_chars_advanced(pns, value); +} + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_float_advanced(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + + static_assert(is_supported_float_type::value, + "only some floating-point types are supported"); + static_assert(is_supported_char_type::value, + "only char, wchar_t, char16_t and char32_t are supported"); + + chars_format const fmt = detail::adjust_for_feature_macros(options.format); + + from_chars_result_t answer; + if (uint64_t(fmt & chars_format::skip_white_space)) { + while ((first != last) && simdjson_fast_float::is_space(*first)) { + first++; + } + } + if (first == last) { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; return answer; } - - static const uint32_t max_shift = 60; - static const uint32_t num_powers = 19; - static const uint8_t powers[19] = { - 0, 3, 6, 9, 13, 16, 19, 23, 26, 29, // - 33, 36, 39, 43, 46, 49, 53, 56, 59, // - }; - int32_t exp2 = 0; - while (d.decimal_point > 0) { - uint32_t n = uint32_t(d.decimal_point); - uint32_t shift = (n < num_powers) ? powers[n] : max_shift; - decimal_right_shift(d, shift); - if (d.decimal_point < -decimal_point_range) { - // should be zero - answer.power2 = 0; - answer.mantissa = 0; + bool const bjf = uint64_t(fmt & detail::basic_json_fmt) != 0; + + // Fast path: parse WITHOUT materializing the integer/fraction spans (read + // only by the rare slow paths). Skipping their stores keeps the fat + // parsed_number_string_t off the hot path. store_spans is a runtime argument, + // so this reuses the single parse_number_string instantiation. + parsed_number_string_t pns = + bjf ? parse_number_string(first, last, options, false) + : parse_number_string(first, last, options, false); + if (!pns.valid) { + if (uint64_t(fmt & chars_format::no_infnan)) { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; return answer; - } - exp2 += int32_t(shift); - } - // We shift left toward [1/2 ... 1]. - while (d.decimal_point <= 0) { - uint32_t shift; - if (d.decimal_point == 0) { - if (d.digits[0] >= 5) { - break; - } - shift = (d.digits[0] < 2) ? 2 : 1; } else { - uint32_t n = uint32_t(-d.decimal_point); - shift = (n < num_powers) ? powers[n] : max_shift; + return detail::parse_infnan(first, last, value, fmt); } - decimal_left_shift(d, shift); - if (d.decimal_point > decimal_point_range) { - // we want to get infinity: - answer.power2 = 0xFF; - answer.mantissa = 0; - return answer; - } - exp2 -= int32_t(shift); } - // We are now in the range [1/2 ... 1] but the binary format uses [1 ... 2]. - exp2--; - constexpr int32_t minimum_exponent = binary::minimum_exponent(); - while ((minimum_exponent + 1) > exp2) { - uint32_t n = uint32_t((minimum_exponent + 1) - exp2); - if (n > max_shift) { - n = max_shift; - } - decimal_right_shift(d, n); - exp2 += int32_t(n); + + // Slow path A (rare): > 19 significant digits. The no-span parse left the + // mantissa un-truncated and skipped the span-based recompute; the cold helper + // re-parses with spans and runs the full algorithm. + // +// We have to disable -Wc++20-extensions for the [[unlikely]] attribute +// See comment for @jwakely at +// https://github.com/fastfloat/simdjson_fast_float/pull/387#discussion_r3366943539 +// This is unfortunate. +#ifdef __clang__ +#pragma clang diagnostic push +#if (!defined(__APPLE_CC__) && __clang_major__ >= 10) || (__clang_major__ >= 13) +#pragma clang diagnostic ignored "-Wc++20-extensions" +#endif +#endif + if simdjson_fastfloat_unlikely (pns.too_many_digits) { + return parse_number_slow_path(first, last, value, options, bjf); } - if ((exp2 - minimum_exponent) >= binary::infinite_power()) { - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; + answer.ec = std::errc(); // be optimistic + answer.ptr = pns.lastmatch; + + if (clinger_fast_path_impl(pns.mantissa, pns.exponent, pns.negative, value)) { return answer; } - const int mantissa_size_in_bits = binary::mantissa_explicit_bits() + 1; - decimal_left_shift(d, mantissa_size_in_bits); - - uint64_t mantissa = round(d); - // It is possible that we have an overflow, in which case we need - // to shift back. - if (mantissa >= (uint64_t(1) << mantissa_size_in_bits)) { - decimal_right_shift(d, 1); - exp2 += 1; - mantissa = round(d); - if ((exp2 - minimum_exponent) >= binary::infinite_power()) { - answer.power2 = binary::infinite_power(); - answer.mantissa = 0; - return answer; - } + adjusted_mantissa am = + compute_float>(pns.exponent, pns.mantissa); + // Slow path B (rare): Eisel-Lemire could not resolve; digit_comp needs the + // integer/fraction spans. Route to the cold helper (clinger there is a + // dead-effect since it already failed here; the cold re-parse + digit_comp + // via from_chars_advanced reproduces this branch). + if simdjson_fastfloat_unlikely (am.power2 < 0) { + return parse_number_slow_path(first, last, value, options, bjf); } - answer.power2 = exp2 - binary::minimum_exponent(); - if (mantissa < (uint64_t(1) << binary::mantissa_explicit_bits())) { - answer.power2--; +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + to_float(pns.negative, am, value); + // Test for over/underflow. + if ((pns.mantissa != 0 && am.mantissa == 0 && am.power2 == 0) || + am.power2 == binary_format::infinite_power()) { + answer.ec = std::errc::result_out_of_range; } - answer.mantissa = - mantissa & ((uint64_t(1) << binary::mantissa_explicit_bits()) - 1); return answer; } -template -adjusted_mantissa parse_long_mantissa(const char *first) { - decimal d = parse_decimal(first); - return compute_float(d); +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars(UC const *first, UC const *last, T &value, int base) noexcept { + + static_assert(is_supported_integer_type::value, + "only integer types are supported"); + static_assert(is_supported_char_type::value, + "only char, wchar_t, char16_t and char32_t are supported"); + + parse_options_t options; + options.base = base; + return from_chars_advanced(first, last, value, options); } -template -adjusted_mantissa parse_long_mantissa(const char *first, const char *end) { - decimal d = parse_decimal(first, end); - return compute_float(d); +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value, T>::type + integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept { + T value; + if (clinger_fast_path_impl(mantissa, decimal_exponent, false, value)) + return value; + + adjusted_mantissa am = + compute_float>(decimal_exponent, mantissa); + to_float(false, am, value); + return value; } -double from_chars(const char *first) noexcept { - bool negative = first[0] == '-'; - if (negative) { - first++; - } - adjusted_mantissa am = parse_long_mantissa>(first); - uint64_t word = am.mantissa; - word |= uint64_t(am.power2) - << binary_format::mantissa_explicit_bits(); - word = negative ? word | (uint64_t(1) << binary_format::sign_index()) - : word; - double value; - std::memcpy(&value, &word, sizeof(double)); +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value, T>::type + integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept { + const bool is_negative = mantissa < 0; + const uint64_t m = static_cast(is_negative ? -mantissa : mantissa); + + T value; + if (clinger_fast_path_impl(m, decimal_exponent, is_negative, value)) + return value; + + adjusted_mantissa am = compute_float>(decimal_exponent, m); + to_float(is_negative, am, value); return value; } +SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double +integer_times_pow10(uint64_t mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(mantissa, decimal_exponent); +} -double from_chars(const char *first, const char *end) noexcept { - bool negative = first[0] == '-'; - if (negative) { - first++; +SIMDJSON_FASTFLOAT_CONSTEXPR20 inline double +integer_times_pow10(int64_t mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(mantissa, decimal_exponent); +} + +// the following overloads are here to avoid surprising ambiguity for int, +// unsigned, etc. +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value && + std::is_integral::value && + !std::is_signed::value, + T>::type + integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(static_cast(mantissa), + decimal_exponent); +} + +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 + typename std::enable_if::value && + std::is_integral::value && + std::is_signed::value, + T>::type + integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(static_cast(mantissa), + decimal_exponent); +} + +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 typename std::enable_if< + std::is_integral::value && !std::is_signed::value, double>::type +integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(static_cast(mantissa), decimal_exponent); +} + +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 typename std::enable_if< + std::is_integral::value && std::is_signed::value, double>::type +integer_times_pow10(Int mantissa, int decimal_exponent) noexcept { + return integer_times_pow10(static_cast(mantissa), decimal_exponent); +} + +template +SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_int_advanced(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + + static_assert(is_supported_integer_type::value, + "only integer types are supported"); + static_assert(is_supported_char_type::value, + "only char, wchar_t, char16_t and char32_t are supported"); + + chars_format const fmt = detail::adjust_for_feature_macros(options.format); + int const base = options.base; + + from_chars_result_t answer; + if (uint64_t(fmt & chars_format::skip_white_space)) { + while ((first != last) && simdjson_fast_float::is_space(*first)) { + first++; + } + } + if (first == last || base < 2 || base > 36) { + answer.ec = std::errc::invalid_argument; + answer.ptr = first; + return answer; + } + + return parse_int_string(first, last, value, options); +} + +template struct from_chars_advanced_caller { + static_assert(TypeIx > 0, "unsupported type"); +}; + +template <> struct from_chars_advanced_caller<1> { + template + simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t + call(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + return from_chars_float_advanced(first, last, value, options); + } +}; + +template <> struct from_chars_advanced_caller<2> { + template + simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 static from_chars_result_t + call(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + return from_chars_int_advanced(first, last, value, options); + } +}; + +template +simdjson_fastfloat_really_inline SIMDJSON_FASTFLOAT_CONSTEXPR20 from_chars_result_t +from_chars_advanced(UC const *first, UC const *last, T &value, + parse_options_t options) noexcept { + return from_chars_advanced_caller< + size_t(is_supported_float_type::value) + + 2 * size_t(is_supported_integer_type::value)>::call(first, last, value, + options); +} + +} // namespace simdjson_fast_float + +#endif + +/* end file simdjson/internal/fast_float.h */ + +#include +#include +#include + +namespace simdjson { +namespace internal { + +/** + * These functions handle floating-point parsing when the fast path in + * numberparsing.h gives up: more than 19 digits in the decimal mantissa, an + * exponent outside the range the power-of-five table covers, or one of the rare + * inputs where the truncated Eisel-Lemire product is not accurate enough to + * round. That should only be seen in adversarial scenarios; we do not expect + * production systems to even produce such floating-point numbers. + * + * The work is handed to fast_float (vendored in + * include/simdjson/internal/fast_float.h), which settles the rounding by + * comparing a bigint against a scaled power of five. It is correctly rounded, + * and quick enough that an adversarial document is no longer worth worrying + * about. + **/ + +namespace { + +// fast_float wants the end of the number, and the callers only promise that a +// number is followed by a character which cannot be part of one -- the input +// has already been validated against the JSON grammar, and the buffer is padded, +// so such a character is always there to be found. Locating it costs a pass over +// digits we are about to parse anyway, and in exchange fast_float can bound its +// inner loops instead of re-checking a far-away end pointer. +const char *find_end_of_number(const char *first) noexcept { + const char *p = first; + while ((*p >= '0' && *p <= '9') || *p == '-' || *p == '+' || *p == '.' || + *p == 'e' || *p == 'E') { + p++; } - adjusted_mantissa am = parse_long_mantissa>(first, end); - uint64_t word = am.mantissa; - word |= uint64_t(am.power2) - << binary_format::mantissa_explicit_bits(); - word = negative ? word | (uint64_t(1) << binary_format::sign_index()) - : word; - double value; - std::memcpy(&value, &word, sizeof(double)); + return p; +} + +// The input is JSON, so parse it under the JSON grammar: no hexadecimal, no +// leading plus, and no infinity or NaN spellings. Those are handled (or +// rejected) before we ever get here. +constexpr simdjson_fast_float::parse_options json_options{ + simdjson_fast_float::chars_format::json}; + +// fast_float reports result_out_of_range for a value at either edge of the +// format, writing +/-0 when it underflows and +/-infinity when it overflows. +// Both are exactly what the callers of these functions expect to receive: they +// accept a zero and treat an infinity as an error. A malformed number cannot +// happen on validated input, but if it somehow did, returning zero matches what +// the previous implementation did with digits it could not use. +template T parse_with_fast_float(const char *first, const char *end) noexcept { + T value{}; + auto answer = + simdjson_fast_float::from_chars_advanced(first, end, value, json_options); + if (answer.ec == std::errc::invalid_argument) { return T(0); } return value; } +} // namespace + +double from_chars(const char *first) noexcept { + return parse_with_fast_float(first, find_end_of_number(first)); +} + +double from_chars(const char *first, const char *end) noexcept { + return parse_with_fast_float(first, end); +} + +float from_chars_float(const char *first) noexcept { + return parse_with_fast_float(first, find_end_of_number(first)); +} + } // internal } // simdjson @@ -5109,7 +10245,8 @@ namespace internal { { SCALAR_DOCUMENT_AS_VALUE, "SCALAR_DOCUMENT_AS_VALUE: A JSON document made of a scalar (number, Boolean, null or string) is treated as a value. Use get_bool(), get_double(), etc. on the document instead. "}, { OUT_OF_BOUNDS, "OUT_OF_BOUNDS: Attempt to access location outside of document."}, { TRAILING_CONTENT, "TRAILING_CONTENT: Unexpected trailing content in the JSON input."}, - { OUT_OF_CAPACITY, "OUT_OF_CAPACITY: The capacity was exceeded, we cannot allocate enough memory."} + { OUT_OF_CAPACITY, "OUT_OF_CAPACITY: The capacity was exceeded, we cannot allocate enough memory."}, + { UNKNOWN_FIELD, "UNKNOWN_FIELD: The JSON object has a field that does not map to any member of the target type (deny_unknown_fields)."} }; // error_messages[] } // namespace internal @@ -7115,7 +12252,12 @@ class document; * 3) The stream_final mode allows us to truncate final * unterminated strings. It is useful in conjunction with streaming_partial. */ -enum class stage1_mode { regular, streaming_partial, streaming_final}; +enum class stage1_mode { + regular, + streaming_partial, streaming_final, + json_sequence_partial, json_sequence_final, + comma_delimited_partial, comma_delimited_final +}; /** * Returns true if mode == streaming_partial or mode == streaming_final @@ -7127,7 +12269,6 @@ inline bool is_streaming(stage1_mode mode) { // return (mode == stage1_mode::streaming_partial || mode == stage1_mode::streaming_final); } - namespace internal { @@ -7312,6 +12453,16 @@ class dom_parser_implementation { /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ bool _number_as_string{false}; + /** + * Whether the input buffer passed to parse() is *not* padded to len + + * SIMDJSON_PADDING bytes. When true, stage 2 string parsing avoids reading + * past buf+len (it finishes the final, near-the-end bytes from a small padded + * scratch buffer). This is set only by the no-padding DOM parse entry points + * (dom::parser::parse_unpadded); the default padded fast path leaves it false + * and is unaffected. + */ + bool _unpadded{false}; + protected: // Declaring these so that subclasses can use them to implement their constructors. @@ -7655,6 +12806,8 @@ enum instruction_set { LASX = 0x40000, //RVV = 0x80000, RVV_VLS = 0x100000, + SVE = 0x200000, + SVE2 = 0x400000, }; } // namespace internal @@ -7966,12 +13119,24 @@ POSSIBILITY OF SUCH DAMAGE. #include #if defined(_MSC_VER) #include -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || defined(__FILC__) #include #endif #if defined(__loongarch__) && defined(__linux__) #include #endif +#if defined(__aarch64__) && defined(__linux__) + #include +#endif +#if (defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) && defined(_WIN32) && !defined(_WINDOWS_) +// We avoid including (macro pollution); this matches the +// declaration in the Windows SDK (BOOL WINAPI IsProcessorFeaturePresent(DWORD)). +extern "C" __declspec(dllimport) int __stdcall IsProcessorFeaturePresent(unsigned long ProcessorFeature); +#endif + +#ifdef __FILC__ +#include +#endif namespace simdjson { namespace internal { @@ -7984,8 +13149,60 @@ static inline uint32_t detect_supported_architectures() { #elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) +#if defined(__linux__) +// The kernel advertises SVE in AT_HWCAP and SVE2 in AT_HWCAP2. Older +// headers may not define these constants, so we provide the kernel's values +// (we deliberately do not include , which is not available on +// all toolchains, e.g., musl without linux-headers). +#ifndef AT_HWCAP2 +#define AT_HWCAP2 26 +#endif +#ifndef HWCAP_SVE +#define HWCAP_SVE (1 << 22) +#endif +#ifndef HWCAP2_SVE2 +#define HWCAP2_SVE2 (1 << 1) +#endif +#endif // __linux__ + +#if defined(_WIN32) +// Only recent Windows SDKs define these processor features. +#ifndef PF_ARM_SVE_INSTRUCTIONS_AVAILABLE +#define PF_ARM_SVE_INSTRUCTIONS_AVAILABLE 46 +#endif +#ifndef PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE +#define PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE 47 +#endif +#endif // _WIN32 + static inline uint32_t detect_supported_architectures() { - return instruction_set::NEON; + // NEON is mandatory on AArch64. + uint32_t host_isa = instruction_set::NEON; +#if defined(__linux__) + unsigned long hwcap = getauxval(AT_HWCAP); + unsigned long hwcap2 = getauxval(AT_HWCAP2); + if (hwcap & HWCAP_SVE) { + host_isa |= instruction_set::SVE; + // We only claim SVE2 when SVE is also present. Before Linux 6.14, the + // kernel set HWCAP2_SVE2 on processors implementing SME(2) but not SVE, + // because SVE2 instructions are available in streaming mode. Our SVE2 + // code runs in non-streaming mode and needs actual SVE. + if (hwcap2 & HWCAP2_SVE2) { + host_isa |= instruction_set::SVE2; + } + } +#elif defined(_WIN32) + if (IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE)) { + host_isa |= instruction_set::SVE; + // As on Linux, require SVE before claiming SVE2. + if (IsProcessorFeaturePresent(PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE)) { + host_isa |= instruction_set::SVE2; + } + } +#endif + // On other systems (e.g., macOS, where Apple Silicon has no SVE), we only + // report NEON. + return host_isa; } #elif defined(__x86_64__) || defined(_M_AMD64) // x64 @@ -8023,7 +13240,7 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, *ebx = cpu_info[1]; *ecx = cpu_info[2]; *edx = cpu_info[3]; -#elif defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID) +#elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || defined(__FILC__) uint32_t level = *eax; __get_cpuid(level, eax, ebx, ecx, edx); #else @@ -8040,6 +13257,8 @@ static inline void cpuid(uint32_t *eax, uint32_t *ebx, uint32_t *ecx, static inline uint64_t xgetbv() { #if defined(_MSC_VER) return _xgetbv(0); +#elif defined(__FILC__) + return zxgetbv(); #else uint32_t xcr0_lo, xcr0_hi; asm volatile("xgetbv\n\t" : "=a" (xcr0_lo), "=d" (xcr0_hi) : "c" (0)); @@ -8601,7 +13820,7 @@ class implementation final : public simdjson::implementation { simdjson_inline implementation() : simdjson::implementation( "rvv_vls", "RISC-V V extension", - 0 + internal::instruction_set::RVV_VLS ) {} simdjson_warn_unused error_code create_dom_parser_implementation( size_t capacity, @@ -8976,7 +14195,14 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { /* result might be undefined when input_num is zero */ simdjson_inline int count_ones(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); +#else + // if the system supports SVE or CSSC, __builtin_popcountll + // might be compiled to fewer single instructions. For CSSC, + // __builtin_popcountll is compiled to a single instruction. + return __builtin_popcountll(input_num); +#endif// SIMDJSON_REGULAR_VISUAL_STUDIO } @@ -9013,15 +14239,6 @@ simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace arm64 @@ -9285,6 +14502,7 @@ namespace { return vget_lane_u64( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); } + // Integer umaxv, not a float compare: flush-to-zero would treat a denormal as zero. simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; @@ -9933,6 +15151,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -9944,6 +15165,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -9980,6 +15223,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace arm64 @@ -10070,7 +15378,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -10249,6 +15557,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -10288,6 +15597,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -10544,6 +15865,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -10581,6 +16115,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -10599,6 +16143,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -10615,26 +16181,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -10723,7 +16332,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -10806,15 +16415,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -10845,7 +16456,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -10894,7 +16519,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -10993,7 +16618,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -11091,7 +16716,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -11146,7 +16771,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -11232,7 +16857,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -11272,11 +16897,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -11287,9 +16921,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -11338,6 +16971,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -11490,11 +17214,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -11505,9 +17243,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -11556,6 +17293,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -11749,6 +17579,9 @@ class implementation final : public simdjson::implementation { #endif // SIMDJSON_ARM64_IMPLEMENTATION_H /* end file simdjson/arm64/implementation.h */ +// defining SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write +#define SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + /* including simdjson/arm64/begin.h: #include */ /* begin file simdjson/arm64/begin.h */ /* defining SIMDJSON_IMPLEMENTATION to "arm64" */ @@ -11861,7 +17694,14 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { /* result might be undefined when input_num is zero */ simdjson_inline int count_ones(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); +#else + // if the system supports SVE or CSSC, __builtin_popcountll + // might be compiled to fewer single instructions. For CSSC, + // __builtin_popcountll is compiled to a single instruction. + return __builtin_popcountll(input_num); +#endif// SIMDJSON_REGULAR_VISUAL_STUDIO } @@ -11898,15 +17738,6 @@ simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace arm64 @@ -12170,6 +18001,7 @@ namespace { return vget_lane_u64( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); } + // Integer umaxv, not a float compare: flush-to-zero would treat a denormal as zero. simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; @@ -13615,6 +19447,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace arm64 @@ -14048,7 +20170,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -14076,6 +20197,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -14158,7 +20348,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -14455,7 +20644,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -14472,6 +20661,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -14480,6 +20670,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -14528,7 +20719,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -14543,7 +20734,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -14555,8 +20746,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -14573,29 +20764,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -14623,16 +20814,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -14660,11 +20851,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -14704,7 +20913,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -14726,7 +20948,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -14912,9 +21147,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -14937,6 +21176,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -14991,73 +21301,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for arm64 */ -/* including generic/stage2/structural_iterator.h for arm64: #include */ -/* begin file generic/stage2/structural_iterator.h for arm64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace arm64 { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace arm64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for arm64 */ /* including generic/stage2/tape_builder.h for arm64: #include */ /* begin file generic/stage2/tape_builder.h for arm64 */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -15080,12 +21323,8 @@ namespace arm64 { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -15138,88 +21377,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -15228,27 +21509,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -15256,7 +21558,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -15270,76 +21573,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -15352,13 +21721,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -15379,6 +21750,38 @@ simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { /* end file generic/stage2/tape_builder.h for arm64 */ /* end file generic/stage2/amalgamated.h for arm64 */ +#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + +namespace simdjson { namespace arm64 { namespace { namespace stage1 { + +// The generic bit_indexer::write emits the structural indexes in groups of +// four (up to 24), so a block with five set bits computes and stores eight +// indexes, three of them wasted. Typical JSON has four to eight structural +// characters per 64-byte block. This version writes the first four indexes +// unconditionally and then continues by groups of two, which cuts the wasted +// work on such blocks at the cost of one more branch for dense blocks. Both +// versions fall back to the same scalar loop past 24 indexes. +simdjson_inline void bit_indexer::write(uint32_t idx, uint64_t bits) { + if (bits == 0) { return; } + + const int cnt = static_cast(count_ones(bits)); +#if SIMDJSON_PREFER_REVERSE_BITS + bits = reverse_bits(bits); +#endif + write_indexes<0, 4>(idx, bits); + if (simdjson_unlikely(4 < cnt)) { + write_indexes_stepped<4, 24, 2>(idx, bits, cnt); + } + if (simdjson_unlikely(24 < cnt)) { + for (int i = 24; i < cnt; ++i) { + write_index(idx, bits, i); + } + } + this->tail += cnt; +} + +}}}} // namespace simdjson::arm64::(anonymous)::stage1 + // // Stage 1 // @@ -15404,55 +21807,43 @@ namespace { using namespace simd; simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // Functional programming causes trouble with Visual Studio. - // Keeping this version in comments since it is much nicer: - // auto v = in.map([&](simd8 chunk) { - // auto nib_lo = chunk & 0xf; - // auto nib_hi = chunk.shr<4>(); - // auto shuf_lo = nib_lo.lookup_16(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); - // auto shuf_hi = nib_hi.lookup_16(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); - // return shuf_lo & shuf_hi; - // }); - const simd8 table1(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); - const simd8 table2(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); - - simd8x64 v( - (in.chunks[0] & 0xf).lookup_16(table1) & (in.chunks[0].shr<4>()).lookup_16(table2), - (in.chunks[1] & 0xf).lookup_16(table1) & (in.chunks[1].shr<4>()).lookup_16(table2), - (in.chunks[2] & 0xf).lookup_16(table1) & (in.chunks[2].shr<4>()).lookup_16(table2), - (in.chunks[3] & 0xf).lookup_16(table1) & (in.chunks[3].shr<4>()).lookup_16(table2) + const uint8x16_t op_table = simd8( + 0xff, 0, ',', ':', 0, '[', ']', '{', '}', 0, 0, 0, 0, 0, 0, 0 + ); + const uint8x16_t ws_table = simd8( + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0, 0xff, 0, 0 ); + const uint8x16_t d0_0 = in.chunks[0]; + const uint8x16_t d0_1 = in.chunks[1]; + const uint8x16_t d0_2 = in.chunks[2]; + const uint8x16_t d0_3 = in.chunks[3]; - // We compute whitespace and op separately. If the code later only use one or the - // other, given the fact that all functions are aggressively inlined, we can - // hope that useless computations will be omitted. This is namely case when - // minifying (we only need whitespace). *However* if we only need spaces, - // it is likely that we will still compute 'v' above with two lookup_16: one - // could do it a bit cheaper. This is in contrast with the x64 implementations - // where we can, efficiently, do the white space and structural matching - // separately. One reason for this difference is that on ARM NEON, the table - // lookups either zero or leave unchanged the characters exceeding 0xF whereas - // on x64, the equivalent instruction (pshufb) automatically applies a mask, - // ignoring the 4 most significant bits. Thus the x64 implementation is - // optimized differently. This being said, if you use this code strictly - // just for minification (or just to identify the structural characters), - // there is a small untaken optimization opportunity here. We deliberately - // do not pick it up. + const uint8x16_t match_op_0 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_0, vdupq_n_u8(3)), 4)), d0_0); + const uint8x16_t match_op_1 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_1, vdupq_n_u8(3)), 4)), d0_1); + const uint8x16_t match_op_2 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_2, vdupq_n_u8(3)), 4)), d0_2); + const uint8x16_t match_op_3 = vceqq_u8(vqtbl1q_u8(op_table, vshrq_n_u8(vaddq_u8(d0_3, vdupq_n_u8(3)), 4)), d0_3); - uint64_t op = simd8x64( - v.chunks[0].any_bits_set(0x7), - v.chunks[1].any_bits_set(0x7), - v.chunks[2].any_bits_set(0x7), - v.chunks[3].any_bits_set(0x7) - ).to_bitmask(); + const uint8x16_t match_ws_0 = vqtbx1q_u8(vceqq_u8(d0_0, vdupq_n_u8(' ')), ws_table, d0_0); + const uint8x16_t match_ws_1 = vqtbx1q_u8(vceqq_u8(d0_1, vdupq_n_u8(' ')), ws_table, d0_1); + const uint8x16_t match_ws_2 = vqtbx1q_u8(vceqq_u8(d0_2, vdupq_n_u8(' ')), ws_table, d0_2); + const uint8x16_t match_ws_3 = vqtbx1q_u8(vceqq_u8(d0_3, vdupq_n_u8(' ')), ws_table, d0_3); - uint64_t whitespace = simd8x64( - v.chunks[0].any_bits_set(0x18), - v.chunks[1].any_bits_set(0x18), - v.chunks[2].any_bits_set(0x18), - v.chunks[3].any_bits_set(0x18) - ).to_bitmask(); + const uint8x16_t bit_mask = simd8( + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80 + ); + + uint8x16_t op_sum0 = vpaddq_u8(vandq_u8(match_op_0, bit_mask), vandq_u8(match_op_1, bit_mask)); + uint8x16_t ws_sum0 = vpaddq_u8(vandq_u8(match_ws_0, bit_mask), vandq_u8(match_ws_1, bit_mask)); + uint8x16_t op_sum1 = vpaddq_u8(vandq_u8(match_op_2, bit_mask), vandq_u8(match_op_3, bit_mask)); + uint8x16_t ws_sum1 = vpaddq_u8(vandq_u8(match_ws_2, bit_mask), vandq_u8(match_ws_3, bit_mask)); + op_sum0 = vpaddq_u8(op_sum0, op_sum1); + ws_sum0 = vpaddq_u8(ws_sum0, ws_sum1); + op_sum0 = vpaddq_u8(op_sum0, op_sum0); + ws_sum0 = vpaddq_u8(ws_sum0, ws_sum0); + const uint64_t op = vgetq_lane_u64(vreinterpretq_u64_u8(op_sum0), 0); + const uint64_t whitespace = vgetq_lane_u64(vreinterpretq_u64_u8(ws_sum0), 0); return { whitespace, op }; } @@ -15498,7 +21889,7 @@ simdjson_warn_unused error_code implementation::minify(const uint8_t *buf, size_ return arm64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); } -simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { +simdjson_flatten simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t *_buf, size_t _len, stage1_mode streaming) noexcept { this->buf = _buf; this->len = _len; return arm64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); @@ -15721,16 +22112,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace haswell @@ -16483,6 +22864,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -16494,6 +22878,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -16530,6 +22936,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace haswell @@ -16620,7 +23091,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -16799,6 +23270,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -16838,6 +23310,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -17094,6 +23578,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -17131,6 +23828,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -17149,6 +23856,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -17165,26 +23894,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -17273,7 +24045,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -17356,15 +24128,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -17395,7 +24169,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -17444,7 +24232,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -17543,7 +24331,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -17641,7 +24429,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -17696,7 +24484,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -17782,7 +24570,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -17822,11 +24610,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -17837,9 +24634,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -17888,6 +24684,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -18040,11 +24927,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -18055,9 +24956,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -18106,6 +25006,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -18465,16 +25458,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace haswell @@ -20024,6 +27007,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace haswell @@ -20457,7 +27730,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -20485,6 +27757,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -20567,7 +27908,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -20864,7 +28204,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -20881,6 +28221,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -20889,6 +28230,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -20937,7 +28279,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -20952,7 +28294,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -20964,8 +28306,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -20982,29 +28324,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -21032,16 +28374,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -21069,11 +28411,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -21113,7 +28473,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -21135,7 +28508,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -21321,9 +28707,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -21346,6 +28736,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -21400,73 +28861,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for haswell */ -/* including generic/stage2/structural_iterator.h for haswell: #include */ -/* begin file generic/stage2/structural_iterator.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace haswell { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for haswell */ /* including generic/stage2/tape_builder.h for haswell: #include */ /* begin file generic/stage2/tape_builder.h for haswell */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -21489,12 +28883,8 @@ namespace haswell { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -21547,88 +28937,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -21637,27 +29069,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -21665,7 +29118,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -21679,76 +29133,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -21761,13 +29281,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -22123,16 +29645,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace icelake @@ -22888,6 +30400,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -22899,6 +30414,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -22935,6 +30472,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace icelake @@ -23025,7 +30627,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -23204,6 +30806,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -23243,6 +30846,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -23499,6 +31114,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -23536,6 +31364,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -23554,6 +31392,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -23570,26 +31430,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -23678,7 +31581,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -23761,15 +31664,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -23800,7 +31705,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -23849,7 +31768,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -23948,7 +31867,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -24046,7 +31965,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -24101,7 +32020,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -24187,7 +32106,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -24227,9 +32146,247 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } p += parse_digit(*p, i); bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } // no integer digits, or 0123 (zero must be solo) if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } @@ -24239,12 +32396,104 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // int64_t exponent = 0; bool overflow; - if (simdjson_likely(*p == '.')) { + if (simdjson_likely((p != src_end) && (*p == '.'))) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits p++; - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -24276,7 +32525,7 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 exponent += exp_neg ? 0-exp : exp; } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + if (*p != '"') { return NUMBER_ERROR; } overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; @@ -24293,163 +32542,39 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { // // Check for minus sign // - bool negative = (*src == '-'); - src += uint8_t(negative); + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; // // Parse the integer part. // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } + return INCORRECT_TYPE; } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -24460,9 +32585,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -24501,9 +32625,9 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // // Assemble (or slow-parse) the float // - double d; + float d; if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } + if (compute_float_32(exponent, i, negative, d)) { return d; } } if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; @@ -24866,16 +32990,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace icelake @@ -26428,6 +34542,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace icelake @@ -26861,7 +35265,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -26889,6 +35292,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -26971,7 +35443,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -27268,7 +35739,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -27285,6 +35756,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -27293,6 +35765,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -27341,7 +35814,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -27356,7 +35829,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -27368,8 +35841,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -27386,29 +35859,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -27436,16 +35909,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -27473,11 +35946,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -27517,7 +36008,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -27539,7 +36043,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -27725,9 +36242,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -27750,6 +36271,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -27804,73 +36396,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for icelake */ -/* including generic/stage2/structural_iterator.h for icelake: #include */ -/* begin file generic/stage2/structural_iterator.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace icelake { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for icelake */ /* including generic/stage2/tape_builder.h for icelake: #include */ /* begin file generic/stage2/tape_builder.h for icelake */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -27893,12 +36418,8 @@ namespace icelake { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -27951,88 +36472,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -28041,27 +36604,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -28069,7 +36653,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -28083,76 +36668,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -28165,13 +36816,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -28542,16 +37195,6 @@ simdjson_inline int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace ppc64 @@ -29450,6 +38093,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -29461,6 +38107,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -29497,6 +38165,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace ppc64 @@ -29587,7 +38320,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -29766,6 +38499,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -29805,6 +38539,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -30061,6 +38807,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -30098,6 +39057,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -30116,6 +39085,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -30132,26 +39123,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -30240,7 +39274,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -30323,15 +39357,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -30362,7 +39398,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -30411,7 +39461,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -30510,7 +39560,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -30608,7 +39658,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -30663,7 +39713,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -30749,7 +39799,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -30789,11 +39839,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -30804,9 +39863,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -30855,6 +39913,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -31007,11 +40156,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -31022,9 +40185,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -31073,6 +40235,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -31398,16 +40653,6 @@ simdjson_inline int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace ppc64 @@ -33103,6 +42348,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace ppc64 @@ -33536,7 +43071,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -33564,6 +43098,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -33646,7 +43249,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -33943,7 +43545,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -33960,6 +43562,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -33968,6 +43571,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -34016,7 +43620,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -34031,7 +43635,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -34043,8 +43647,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -34061,29 +43665,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -34111,16 +43715,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -34148,11 +43752,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -34192,7 +43814,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -34214,7 +43849,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -34400,9 +44048,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -34425,6 +44077,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -34479,73 +44202,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for ppc64 */ -/* including generic/stage2/structural_iterator.h for ppc64: #include */ -/* begin file generic/stage2/structural_iterator.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace ppc64 { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for ppc64 */ /* including generic/stage2/tape_builder.h for ppc64: #include */ /* begin file generic/stage2/tape_builder.h for ppc64 */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -34568,12 +44224,8 @@ namespace ppc64 { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -34626,88 +44278,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -34716,27 +44410,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -34744,7 +44459,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -34758,76 +44474,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -34840,13 +44622,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -35161,16 +44945,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -35749,16 +45523,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -36372,6 +46136,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -36383,6 +46150,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -36419,6 +46208,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace westmere @@ -36509,7 +46363,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -36688,6 +46542,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -36727,6 +46582,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -36983,6 +46850,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -37020,6 +47100,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -37038,6 +47128,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -37054,26 +47166,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -37162,7 +47317,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -37245,15 +47400,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -37284,7 +47441,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -37333,7 +47504,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -37432,7 +47603,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37530,7 +47701,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37585,7 +47756,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37671,7 +47842,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37711,9 +47882,247 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } p += parse_digit(*p, i); bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } // no integer digits, or 0123 (zero must be solo) if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } @@ -37723,12 +48132,104 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // int64_t exponent = 0; bool overflow; - if (simdjson_likely(*p == '.')) { + if (simdjson_likely((p != src_end) && (*p == '.'))) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits p++; - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -37760,7 +48261,7 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 exponent += exp_neg ? 0-exp : exp; } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + if (*p != '"') { return NUMBER_ERROR; } overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; @@ -37777,163 +48278,39 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { // // Check for minus sign // - bool negative = (*src == '-'); - src += uint8_t(negative); + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; // // Parse the integer part. // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; + return INCORRECT_TYPE; } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -37944,9 +48321,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -37985,9 +48361,9 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // // Assemble (or slow-parse) the float // - double d; + float d; if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } + if (compute_float_32(exponent, i, negative, d)) { return d; } } if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; @@ -38332,16 +48708,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -38920,16 +49286,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -40340,6 +50696,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace westmere @@ -40773,7 +51419,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -40801,6 +51446,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -40883,7 +51597,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -41180,7 +51893,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -41197,6 +51910,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -41205,6 +51919,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -41253,7 +51968,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -41268,7 +51983,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -41280,8 +51995,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -41298,29 +52013,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -41348,16 +52063,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -41385,11 +52100,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -41429,7 +52162,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -41451,7 +52197,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -41637,9 +52396,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -41662,6 +52425,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -41716,73 +52550,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for westmere */ -/* including generic/stage2/structural_iterator.h for westmere: #include */ -/* begin file generic/stage2/structural_iterator.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace westmere { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for westmere */ /* including generic/stage2/tape_builder.h for westmere: #include */ /* begin file generic/stage2/tape_builder.h for westmere */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -41805,12 +52572,8 @@ namespace westmere { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -41863,88 +52626,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -41953,27 +52758,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -41981,7 +52807,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -41995,76 +52822,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -42077,13 +52970,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -42390,10 +53285,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{input_num, 0, 0, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lasx @@ -43140,6 +54031,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -43151,6 +54045,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -43187,6 +54103,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace lasx @@ -43277,7 +54258,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -43456,6 +54437,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -43495,6 +54477,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -43751,6 +54745,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -43788,6 +54995,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -43806,6 +55023,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -43822,26 +55061,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -43930,7 +55212,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -44013,15 +55295,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -44052,7 +55336,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -44101,7 +55399,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -44200,7 +55498,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -44298,7 +55596,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -44353,7 +55651,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -44439,7 +55737,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -44479,11 +55777,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -44494,9 +55801,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -44545,6 +55851,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -44697,11 +56094,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -44712,9 +56123,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -44763,6 +56173,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -45061,10 +56564,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{input_num, 0, 0, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lasx @@ -46608,6 +58107,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace lasx @@ -47041,7 +58830,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -47069,6 +58857,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -47151,7 +59008,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -47448,7 +59304,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -47465,6 +59321,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -47473,6 +59330,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -47521,7 +59379,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -47536,7 +59394,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -47548,8 +59406,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -47566,29 +59424,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -47616,16 +59474,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -47653,11 +59511,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -47697,7 +59573,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -47719,7 +59608,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -47905,9 +59807,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -47930,6 +59836,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -47984,73 +59961,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for lasx */ -/* including generic/stage2/structural_iterator.h for lasx: #include */ -/* begin file generic/stage2/structural_iterator.h for lasx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lasx { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace lasx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for lasx */ /* including generic/stage2/tape_builder.h for lasx: #include */ /* begin file generic/stage2/tape_builder.h for lasx */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -48073,12 +59983,8 @@ namespace lasx { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -48131,88 +60037,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -48221,27 +60169,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -48249,7 +60218,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -48263,76 +60233,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -48345,13 +60381,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -48613,10 +60651,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{input_num, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lsx @@ -49345,6 +61379,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -49356,6 +61393,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -49392,6 +61451,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace lsx @@ -49482,7 +61606,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -49661,6 +61785,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -49700,6 +61825,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -49956,6 +62093,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -49993,6 +62343,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -50011,6 +62371,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -50027,26 +62409,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -50135,7 +62560,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -50218,15 +62643,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -50257,7 +62684,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -50306,7 +62747,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -50405,7 +62846,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -50503,7 +62944,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -50558,7 +62999,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -50644,7 +63085,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -50684,9 +63125,247 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } p += parse_digit(*p, i); bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } // no integer digits, or 0123 (zero must be solo) if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } @@ -50696,12 +63375,104 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // int64_t exponent = 0; bool overflow; - if (simdjson_likely(*p == '.')) { + if (simdjson_likely((p != src_end) && (*p == '.'))) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits p++; - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -50733,7 +63504,7 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 exponent += exp_neg ? 0-exp : exp; } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + if (*p != '"') { return NUMBER_ERROR; } overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; @@ -50750,163 +63521,39 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { // // Check for minus sign // - bool negative = (*src == '-'); - src += uint8_t(negative); + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; // // Parse the integer part. // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif - exponent += exp_neg ? 0-exp : exp; + return INCORRECT_TYPE; } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; - } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -50917,9 +63564,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -50958,9 +63604,9 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // // Assemble (or slow-parse) the float // - double d; + float d; if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } + if (compute_float_32(exponent, i, negative, d)) { return d; } } if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; @@ -51251,10 +63897,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{input_num, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lsx @@ -52780,6 +65422,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace lsx @@ -53213,7 +66145,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -53241,6 +66172,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -53323,7 +66323,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -53620,7 +66619,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -53637,6 +66636,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -53645,6 +66645,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -53693,7 +66694,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -53708,7 +66709,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -53720,8 +66721,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -53738,29 +66739,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -53788,16 +66789,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -53825,11 +66826,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -53869,7 +66888,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -53891,7 +66923,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -54077,9 +67122,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -54102,6 +67151,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -54156,73 +67276,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for lsx */ -/* including generic/stage2/structural_iterator.h for lsx: #include */ -/* begin file generic/stage2/structural_iterator.h for lsx */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace lsx { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace lsx -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for lsx */ /* including generic/stage2/tape_builder.h for lsx: #include */ /* begin file generic/stage2/tape_builder.h for lsx */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -54245,12 +67298,8 @@ namespace lsx { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -54303,88 +67352,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -54393,27 +67484,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -54421,7 +67533,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -54435,76 +67548,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -54517,13 +67696,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -54793,11 +67974,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { return __builtin_popcountll(input_num); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace rvv_vls @@ -55538,6 +68714,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -55549,6 +68728,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -55585,6 +68786,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace rvv_vls @@ -55675,7 +68941,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -55854,6 +69120,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -55893,6 +69160,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -56149,6 +69428,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -56186,6 +69678,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -56204,6 +69706,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -56220,26 +69744,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -56328,7 +69895,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -56411,15 +69978,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -56450,7 +70019,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -56499,7 +70082,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -56598,7 +70181,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -56696,7 +70279,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -56751,7 +70334,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -56837,7 +70420,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -56877,11 +70460,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -56892,9 +70484,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -56943,6 +70534,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -57095,11 +70777,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -57110,9 +70806,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -57161,6 +70856,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -57339,7 +71127,7 @@ class implementation final : public simdjson::implementation { simdjson_inline implementation() : simdjson::implementation( "rvv_vls", "RISC-V V extension", - 0 + internal::instruction_set::RVV_VLS ) {} simdjson_warn_unused error_code create_dom_parser_implementation( size_t capacity, @@ -57456,11 +71244,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { return __builtin_popcountll(input_num); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace rvv_vls @@ -59371,6 +73154,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace rvv_vls @@ -59804,7 +73877,6 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa return EMPTY; } } - parser.n_structural_indexes = new_structural_indexes; } else if (partial == stage1_mode::streaming_final) { if(have_unclosed_string) { parser.n_structural_indexes--; } @@ -59832,6 +73904,75 @@ simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementa // the trailing garbage. return EMPTY; } + } else if (partial == stage1_mode::json_sequence_partial) { + // RFC 7464: use RS positions for batch boundaries + // A discarded unclosed string also caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = find_next_document_index_json_sequence(parser, len, false, next_batch_start, scan_len); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::json_sequence_final) { + // RFC 7464: final batch, last document extends to EOF + // As above: a discarded unclosed string caps how far the filter may scan. + size_t scan_len = len; + if(have_unclosed_string) { + parser.n_structural_indexes--; + scan_len = parser.structural_indexes[parser.n_structural_indexes]; + } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = find_next_document_index_json_sequence(parser, len, true, next_batch_start, scan_len); + // The filter compacted structural_indexes in place and restored the EOF + // sentinel past the compacted end, so the copy below is either the start + // of a truncated document or len, as in streaming_final. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } + } else if (partial == stage1_mode::comma_delimited_partial) { + // Comma-delimited: filter root-level commas, use comma positions for batch boundaries + if(have_unclosed_string) { + parser.n_structural_indexes--; + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + uint32_t next_batch_start = uint32_t(len); + auto new_structural_indexes = filter_comma_delimited(parser, len, false, next_batch_start); + if (new_structural_indexes == DOCUMENT_TOO_LARGE) { + return CAPACITY; + } + if (new_structural_indexes == 0) { + // An EMPTY batch must still advance next_batch_start, or document_stream + // re-parses the same bytes forever. CAPACITY when it cannot. + if (next_batch_start == 0) { return CAPACITY; } + parser.n_structural_indexes = 0; + parser.structural_indexes[0] = next_batch_start; + return EMPTY; + } + parser.n_structural_indexes = new_structural_indexes; + parser.structural_indexes[parser.n_structural_indexes] = next_batch_start; + } else if (partial == stage1_mode::comma_delimited_final) { + // Comma-delimited: final batch, last document extends to EOF + if(have_unclosed_string) { parser.n_structural_indexes--; } + uint32_t next_batch_start = uint32_t(len); + parser.n_structural_indexes = filter_comma_delimited(parser, len, true, next_batch_start); + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return EMPTY; } } checker.check_eof(); return checker.errors(); @@ -59914,7 +74055,6 @@ namespace { namespace stage2 { class json_iterator; -class structural_iterator; struct tape_builder; struct tape_writer; @@ -60211,7 +74351,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -60228,6 +74368,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -60236,6 +74377,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -60284,7 +74426,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -60299,7 +74441,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -60311,8 +74453,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -60329,29 +74471,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -60379,16 +74521,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -60416,11 +74558,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -60460,7 +74620,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -60482,7 +74655,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -60668,9 +74854,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -60693,6 +74883,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -60747,73 +75008,6 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t #endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H /* end file generic/stage2/stringparsing.h for rvv_vls */ -/* including generic/stage2/structural_iterator.h for rvv_vls: #include */ -/* begin file generic/stage2/structural_iterator.h for rvv_vls */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { -namespace rvv_vls { -namespace { -namespace stage2 { - -class structural_iterator { -public: - const uint8_t* const buf; - uint32_t *next_structural; - dom_parser_implementation &dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation &_dom_parser, size_t start_structural_index) - : buf{_dom_parser.buf}, - next_structural{&_dom_parser.structural_indexes[start_structural_index]}, - dom_parser{_dom_parser} { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural-1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural-1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural-1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } -}; - -} // namespace stage2 -} // unnamed namespace -} // namespace rvv_vls -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for rvv_vls */ /* including generic/stage2/tape_builder.h for rvv_vls: #include */ /* begin file generic/stage2/tape_builder.h for rvv_vls */ #ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H @@ -60836,12 +75030,8 @@ namespace rvv_vls { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -60894,88 +75084,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -60984,27 +75216,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -61012,7 +75265,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -61026,76 +75280,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -61108,13 +75428,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -61708,6 +76030,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -61719,6 +76044,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -61755,6 +76102,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace fallback @@ -61845,7 +76257,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -62024,6 +76436,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -62063,6 +76476,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -62319,6 +76744,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -62356,6 +76994,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -62374,6 +77022,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -62390,26 +77060,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -62498,7 +77211,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -62581,15 +77294,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -62620,7 +77335,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -62669,7 +77398,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -62768,7 +77497,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -62866,7 +77595,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -62921,7 +77650,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -63007,7 +77736,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -63047,9 +77776,247 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } p += parse_digit(*p, i); bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } // no integer digits, or 0123 (zero must be solo) if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } @@ -63059,12 +78026,104 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // int64_t exponent = 0; bool overflow; - if (simdjson_likely(*p == '.')) { + if (simdjson_likely((p != src_end) && (*p == '.'))) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits p++; - while (parse_digit(*p, i)) { p++; } + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -63096,7 +78155,7 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 exponent += exp_neg ? 0-exp : exp; } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + if (*p != '"') { return NUMBER_ERROR; } overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; @@ -63113,163 +78172,39 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { // // Check for minus sign // - bool negative = (*src == '-'); - src += uint8_t(negative); + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; // // Parse the integer part. // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; + return INCORRECT_TYPE; } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -63280,9 +78215,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -63321,9 +78255,9 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // // Assemble (or slow-parse) the float // - double d; + float d; if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } + if (compute_float_32(exponent, i, negative, d)) { return d; } } if (!parse_float_fallback(src - uint8_t(negative), &d)) { return NUMBER_ERROR; @@ -63871,6 +78805,296 @@ simdjson_inline uint32_t find_next_document_index(dom_parser_implementation &par return 0; } +/** + * Sentinel value returned to indicate a document started but didn't fit + * (CAPACITY error), as opposed to 0 which means no document content found + * (EMPTY). + */ +constexpr uint32_t DOCUMENT_TOO_LARGE = UINT32_MAX; + +/** + * For RFC 7464 JSON text sequences, filter RS from structural indexes and + * find batch boundaries. + * + * In JSON sequence mode, RS (0x1E) marks the start of each JSON text. + * RS bytes appear in structural_indexes as they are classified as scalars. + * This function: + * 1. Scans structural_indexes to find and count RS positions + * 2. Filters RS out of structural_indexes in-place + * 3. Determines batch boundaries based on RS positions + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @param scan_len Offset past which no value start may be derived. Equals len + * unless stage 1 dropped a trailing unclosed string, whose bytes it + * never validated. + * @return The number of structural indexes to keep (after RS filtering), + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t find_next_document_index_json_sequence( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start, + size_t scan_len) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Phase 1: Scan structural_indexes to find RS positions and handle them. + // RS marks the start of a JSON text. For objects/arrays, the '{' or '[' after RS + // is already in structural_indexes (it's an operator). For scalars like numbers, + // the digit following RS is NOT in structural_indexes because the scanner sees + // RS as a scalar, making the digit a scalar continuation, not a start. + // We must: (1) remove RS from structural_indexes, and (2) for scalars, add the + // actual value start position. + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_rs_pos = 0; + uint32_t rs_count = 0; + + for (uint32_t read_idx = 0; read_idx < parser.n_structural_indexes; read_idx++) { + const uint32_t pos = parser.structural_indexes[read_idx]; + if (parser.buf[pos] == 0x1E) { + // This is an RS character - find the actual JSON value start. + last_rs_pos = pos; + rs_count++; + // Skip past this RS and any whitespace *and any additional RSes* + // to locate the real value. Consecutive RSes are degenerate + // "empty records" per RFC 7464; we collapse them here. They do + // not always appear as separate entries in structural_indexes + // because the scanner groups runs of adjacent non-whitespace + // scalar bytes (including RS) into a single scalar start. + uint32_t value_start = pos + 1; + while (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + if (c == ' ' || c == '\t' || c == '\n' || c == '\r') { + value_start++; + } else if (c == 0x1E) { + // Collapsed empty record. Still count it so rs_count reflects + // the true number of record markers and last_rs_pos tracks + // the final one. + last_rs_pos = value_start; + rs_count++; + value_start++; + } else { + break; + } + } + // If the scanner emitted additional structurals inside the + // whitespace+RS run we just walked over (i.e., isolated RSes + // separated by whitespace), skip past them so we do not + // double-count or double-emit. + while (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] < value_start) { + read_idx++; + } + // Check if the value start is an operator (always present in + // scanner structural_indexes) or a scalar-like start (which may + // be missing from structural_indexes and must be added here). + // Note: '"' is NOT always in structural_indexes. The scanner + // classifies '"' as a scalar character and emits it as a + // structural only when it is a *scalar start* (preceded by + // whitespace or an operator). When '"' immediately follows an + // RS (which the scanner also classifies as scalar), it is + // treated as a scalar continuation and not emitted - so we + // must add it here just like any other scalar value. + if (value_start < scan_len) { + const uint8_t c = parser.buf[value_start]; + const bool is_operator = + (c == '{' || c == '}' || c == '[' || c == ']' || + c == ':' || c == ','); + // If the next scanner structural is exactly at value_start, + // the scanner already emitted it (it followed whitespace) and + // we must not add a duplicate - a subsequent iteration will + // copy it into write_idx. + const bool already_emitted = + (read_idx + 1 < parser.n_structural_indexes && + parser.structural_indexes[read_idx + 1] == value_start); + if (!is_operator && !already_emitted) { + // Scalar value (number/true/false/null/string) - add its + // position since scanner missed it. + parser.structural_indexes[write_idx++] = value_start; + } + } + } else { + // Not RS, copy to output + parser.structural_indexes[write_idx++] = pos; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { + // Only RS markers here: the last one opens a record continuing past the + // window, so that is where the next batch resumes. + if (!is_final && rs_count > 0) { next_batch_start = last_rs_pos; } + return 0; + } + if (rs_count == 0) { + // No RS found; for final batch, try generic boundary detection + return is_final ? find_next_document_index(parser) : 0; + } + + // Phase 2: Determine batch boundaries based on RS positions + + if (is_final) { + // Final batch: all documents are complete (last one ends at EOF). + // In json_sequence mode, RS markers define document boundaries, so all + // remaining structurals form complete documents. Return them all directly. + // (Calling find_next_document_index() would fail for scalar documents.) + return parser.n_structural_indexes; + } + + // Partial batch: need to find complete documents only. + // A document starting at an RS is complete if there is another RS after it. + next_batch_start = last_rs_pos; + + if (rs_count < 2) { + // Only one RS, so we have at most one document that may be incomplete. + // We cannot confirm it is complete without another RS. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only separators. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least 2 RS markers. The last complete document ends before last_rs_pos. + + // Find the structural index cutoff: keep only structurals < last_rs_pos. + // Since we already filtered RS, all remaining structurals are valid. + // We iterate backward to find the last structural before last_rs_pos. + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_rs_pos) { + keep_count = i; + break; + } + } + + // No structurals before the last RS - no complete documents + if (keep_count == 0) { return 0; } + + // All documents before the last RS are complete by definition (the next RS + // confirms their end). No need to call find_next_document_index() which + // would fail for scalar documents like `1` or `"hello"`. + return keep_count; +} + +/** + * Filter comma-delimited documents by removing root-level commas from + * structural indexes. + * + * For comma-delimited format like `{...},{...},{...}`, we need to remove + * the commas that separate documents (depth 0) while preserving commas + * inside arrays and objects (depth > 0). + * + * After filtering, the structural indexes look like whitespace-delimited + * documents, so find_next_document_index() works unchanged. + * + * @param parser The parser with structural_indexes and buf. + * @param len The length of the current batch buffer. + * @param is_final True if this is the final batch (no more data coming). + * @param next_batch_start Output: offset where the next batch should start. + * @return The number of structural indexes to keep, + * 0 if no document content found (EMPTY), + * or DOCUMENT_TOO_LARGE if a document started but didn't fit (CAPACITY). + */ +simdjson_inline uint32_t filter_comma_delimited( + dom_parser_implementation &parser, + size_t len, + bool is_final, + uint32_t &next_batch_start) { + // Default: next batch starts at end of buffer + next_batch_start = uint32_t(len); + + if (parser.n_structural_indexes == 0) { return 0; } + + // Track depth to identify root-level commas (depth 0) + int depth = 0; + // The EOF sentinel: len, or where a discarded unclosed string starts. + const uint32_t sentinel = parser.structural_indexes[parser.n_structural_indexes]; + uint32_t write_idx = 0; + uint32_t last_root_comma_pos = 0; + uint32_t root_comma_count = 0; + + for (uint32_t i = 0; i < parser.n_structural_indexes; i++) { + uint32_t idx = parser.structural_indexes[i]; + uint8_t c = parser.buf[idx]; + + switch (c) { + case '{': case '[': + depth++; + parser.structural_indexes[write_idx++] = idx; + break; + case '}': case ']': + depth--; + parser.structural_indexes[write_idx++] = idx; + break; + case ',': + if (depth == 0) { + // Root-level comma = document boundary, skip it + last_root_comma_pos = idx; + root_comma_count++; + continue; + } + parser.structural_indexes[write_idx++] = idx; + break; + default: + // Colons, scalars, etc. + parser.structural_indexes[write_idx++] = idx; + break; + } + } + + // Update structural index count. Compaction left a stale index in the slot + // past the end: restore the EOF sentinel that stage 1 had planted there, which + // document_stream::truncated_bytes() reads after a final batch. + parser.n_structural_indexes = write_idx; + parser.structural_indexes[write_idx] = sentinel; + + if (parser.n_structural_indexes == 0) { return 0; } + + if (is_final) { + // Final batch: use standard boundary detection on filtered indexes + return find_next_document_index(parser); + } + + // Partial batch: need to find complete documents only. + // A document ending with a root comma is complete. + if (root_comma_count == 0) { + // No root commas found; we cannot confirm any document is complete. + // The whole batch might be one incomplete document. + // Return DOCUMENT_TOO_LARGE if content was found (write_idx > 0), 0 if only commas. + return (parser.n_structural_indexes > 0) ? DOCUMENT_TOO_LARGE : 0; + } + + // We have at least one root comma. Documents before the last comma are complete. + next_batch_start = last_root_comma_pos + 1; + + // Find the structural index cutoff: keep only structurals < last_root_comma_pos + uint32_t keep_count = 0; + for (uint32_t i = parser.n_structural_indexes; i > 0; i--) { + if (parser.structural_indexes[i - 1] < last_root_comma_pos) { + keep_count = i; + break; + } + } + + if (keep_count == 0) { return 0; } + + // Use standard boundary detection on the complete portion + parser.n_structural_indexes = keep_count; + return find_next_document_index(parser); +} + } // namespace stage1 } // unnamed namespace } // namespace fallback @@ -64050,9 +79274,13 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u within the unicode codepoint handling code. */ src += bs_dist; dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } + // Decode adjacent Unicode escapes without returning to the + // quote-and-backslash scanner between code points. + do { + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } while (src[0] == '\\' && src[1] == 'u'); } else { /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and * write bs_dist+1 characters to output @@ -64075,6 +79303,77 @@ simdjson_warn_unused simdjson_inline uint8_t *parse_string(const uint8_t *src, u } } +/** + * Bounds-safe variant of parse_string for input buffers that are NOT padded to + * len + SIMDJSON_PADDING bytes. `buf_end` is one past the last readable input + * byte (buf + len). It behaves exactly like parse_string while we are at least + * SIMDJSON_PADDING bytes away from buf_end (so every speculative SIMD read stays + * in bounds); once within the final SIMDJSON_PADDING bytes it copies the few + * remaining bytes into a space-padded scratch buffer and finishes with the + * regular parse_string. This keeps the delicate escape/Unicode handling in one + * place (the proven parse_string) rather than duplicating it. + * + * Correctness relies on stage 1 having validated the string, i.e. there is an + * unescaped closing quote within [src, buf_end); that quote is therefore inside + * the copied scratch, so parse_string finds it without running off the scratch. + */ +simdjson_warn_unused simdjson_inline uint8_t *parse_string_safe(const uint8_t *src, uint8_t *dst, bool allow_replacement, const uint8_t *buf_end) { + // Far from the end: identical to parse_string's loop. The guard uses + // SIMDJSON_PADDING (>= BYTES_PROCESSED) so copy_and_find never reads past + // buf_end; escape/Unicode look-aheads read within the string (before the + // closing quote, which is < buf_end), so they are in bounds here too. + // We add margin (+12) for handle_unicode_codepoint's worst-case lookahead: + // after seeing a high surrogate, it does hex_to_u32_nocheck on the immediate + // following bytes (+6 from the '\'), then (if it sees \u) another + // hex_to_u32_nocheck at +8..+11 relative to the backslash that started the + // escape. With bs_dist up to BYTES_PROCESSED-1 this reaches +11 from the + // chunk start. The +12 margin ensures that even on kernels where + // BYTES_PROCESSED == SIMDJSON_PADDING (e.g. icelake) the 4-byte read stays + // in-bounds. The scratch fallback (3*PAD) is already safe. + while (src + SIMDJSON_PADDING + 12 <= buf_end) { + auto b = backslash_and_quote{}; + auto bs_quote = b.copy_and_find(src, dst); + if (bs_quote.has_quote_first()) { + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + if (escape_char == 'u') { + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } else { + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } else { + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + // Within the final SIMDJSON_PADDING bytes: copy what remains into a + // space-padded scratch (spaces are neither quote nor backslash, so they do not + // disturb matching) and let the regular parser finish from there. The closing + // quote is within `remaining` (< SIMDJSON_PADDING), so parse_string finds it in + // the chunk starting at some offset <= remaining and reads at most + // BYTES_PROCESSED (<= SIMDJSON_PADDING) further -- i.e. under 2*SIMDJSON_PADDING. + // We size at 3x for a comfortable margin (the unicode look-ahead reads a few + // extra bytes past an escape). + uint8_t scratch[SIMDJSON_PADDING * 3]; + const size_t remaining = size_t(buf_end - src); // < SIMDJSON_PADDING + std::memset(scratch, ' ', sizeof(scratch)); + std::memcpy(scratch, src, remaining); + return parse_string(scratch, dst, allow_replacement); +} + simdjson_warn_unused simdjson_inline uint8_t *parse_wobbly_string(const uint8_t *src, uint8_t *dst) { // It is not ideal that this function is nearly identical to parse_string. while (1) { @@ -64278,7 +79577,7 @@ class json_iterator { * * - increment_count(iter) - each time a value is found in an array or object. */ - template + template simdjson_warn_unused simdjson_inline error_code walk_document(V &visitor) noexcept; /** @@ -64295,6 +79594,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *peek() const noexcept; /** * Advance to the next token. @@ -64303,6 +79603,7 @@ class json_iterator { * * They may include invalid JSON as well (such as `1.2.3` or `ture`). */ + template simdjson_inline const uint8_t *advance() noexcept; /** * Get the remaining length of the document, from the start of the current token. @@ -64351,7 +79652,7 @@ class json_iterator { simdjson_warn_unused simdjson_inline error_code visit_primitive(V &visitor, const uint8_t *value) noexcept; }; -template +template simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V &visitor) noexcept { logger::log_start(); @@ -64366,7 +79667,7 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & // Read first value // { - auto value = advance(); + auto value = advance(); // Make sure the outer object or array is closed before continuing; otherwise, there are ways we // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 @@ -64378,8 +79679,8 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & } switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_root_primitive(*this, value) ); break; } } @@ -64396,29 +79697,29 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & SIMDJSON_TRY( visitor.visit_object_start(*this) ); { - auto key = advance(); + auto key = advance(); if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.increment_count(*this) ); SIMDJSON_TRY( visitor.visit_key(*this, key) ); } object_field: - if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + if (simdjson_unlikely( *advance() != ':' )) { log_error("Missing colon after key in object"); return TAPE_ERROR; } { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } object_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); { - auto key = advance(); + auto key = advance(); if (simdjson_unlikely( *key != '"' )) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } SIMDJSON_TRY( visitor.visit_key(*this, key) ); } @@ -64446,16 +79747,16 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V & array_value: { - auto value = advance(); + auto value = advance(); switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY( visitor.visit_empty_object(*this) ); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY( visitor.visit_empty_array(*this) ); break; } goto array_begin; default: SIMDJSON_TRY( visitor.visit_primitive(*this, value) ); break; } } array_continue: - switch (*advance()) { + switch (*advance()) { case ',': SIMDJSON_TRY( visitor.increment_count(*this) ); goto array_value; case ']': log_end_value("array"); SIMDJSON_TRY( visitor.visit_array_end(*this) ); goto scope_end; default: log_error("Missing comma between array values"); return TAPE_ERROR; @@ -64483,11 +79784,29 @@ simdjson_inline json_iterator::json_iterator(dom_parser_implementation &_dom_par dom_parser{_dom_parser} { } +// Stage 1 leaves a sentinel index of value `len`; reading buf[len] requires +// padding. For UNPADDED, return a dummy so walk_document can fail cleanly (#2815). +template simdjson_inline const uint8_t *json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; + const uint32_t idx = *(next_structural); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } +template simdjson_inline const uint8_t *json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; + const uint32_t idx = *(next_structural++); + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + if (simdjson_unlikely(idx >= dom_parser.len)) { + static constexpr uint8_t unpadded_eof_sentinel = 0; + return &unpadded_eof_sentinel; + } + } + return &buf[idx]; } simdjson_inline size_t json_iterator::remaining_len() const noexcept { return dom_parser.len - *(next_structural-1); @@ -64527,7 +79846,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primit case '"': return visitor.visit_root_string(*this, value); case 't': return visitor.visit_root_true_atom(*this, value); case 'f': return visitor.visit_root_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_root_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_root_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_root_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_root_inf_atom(*this, value); +#else case 'n': return visitor.visit_root_null_atom(*this, value); +#endif case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -64549,7 +79881,20 @@ simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V switch (*value) { case 't': return visitor.visit_true_atom(*this, value); case 'f': return visitor.visit_false_atom(*this, value); +#if SIMDJSON_ENABLE_NAN_INF + case 'n': { + auto err = visitor.visit_null_atom(*this, value); + if (err == SUCCESS) { return err; } + // propagate the error value returned by a bad 'null' atom if parsing 'nan' fails + return visitor.visit_nan_atom(*this, value, err); + } + // 'N' isn't a canonically recognized atom, so we return a TAPE_ERROR if failure occurs + case 'N': return visitor.visit_nan_atom(*this, value, TAPE_ERROR); + case 'i': + case 'I': return visitor.visit_inf_atom(*this, value); +#else case 'n': return visitor.visit_null_atom(*this, value); +#endif default: log_error("Non-value found when value was expected!"); return TAPE_ERROR; @@ -64720,12 +80065,8 @@ namespace fallback { namespace { namespace stage2 { -struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept; - +template +struct tape_builder_impl { /** Called when a non-empty document starts. */ simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator &iter) noexcept; /** Called when a non-empty document ends without error. */ @@ -64778,88 +80119,130 @@ struct tape_builder { simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept; simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept; +#if SIMDJSON_ENABLE_NAN_INF + simdjson_warn_unused simdjson_inline error_code visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept; + // Attempts to parse 'inf' or 'infinity' (case insensitive). Because neither are canonical atoms, + // this returns a tape error on failure. + simdjson_warn_unused simdjson_inline error_code visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept; +#endif + /** Called each time a new field or element in an array or object is found. */ simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator &iter) noexcept; /** Next location to write to tape */ tape_writer tape; +public: + simdjson_inline tape_builder_impl(dom::document &doc) noexcept; private: /** Next write location in the string buf for stage 2 parsing */ uint8_t *current_string_buf_loc; - simdjson_inline tape_builder(dom::document &doc) noexcept; - simdjson_inline uint32_t next_tape_index(json_iterator &iter) const noexcept; simdjson_inline void start_container(json_iterator &iter) noexcept; simdjson_warn_unused simdjson_inline error_code end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept; simdjson_inline uint8_t *on_start_string(json_iterator &iter) noexcept; simdjson_inline void on_end_string(uint8_t *dst) noexcept; -}; // struct tape_builder +}; // struct tape_builder_impl -template -simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation &dom_parser, - dom::document &doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); -} +// Thin, non-templated entry so each architecture's stage2() keeps calling +// tape_builder::parse_document unchanged. It chooses the bounds-safe +// (unpadded) or the regular (padded) tape_builder_impl ONCE per document, so the +// choice is a compile-time constant inside the walk: the padded path carries no +// extra branch or load (see tape_builder_impl::visit_string). +struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation &dom_parser, dom::document &doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + if (dom_parser._unpadded) { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } else { + tape_builder_impl builder(doc); + return iter.walk_document(builder); + } + } +}; -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_root_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_primitive(json_iterator &iter, const uint8_t *value) noexcept { return iter.visit_primitive(*this, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_object(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_empty_array(json_iterator &iter) noexcept { return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_start(json_iterator &iter) noexcept { start_container(iter); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_object_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_array_end(json_iterator &iter) noexcept { return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_document_end(json_iterator &iter) noexcept { constexpr uint32_t start_tape_index = 0; tape.append(start_tape_index, internal::tape_type::ROOT); tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator &iter, const uint8_t *key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_key(json_iterator &iter, const uint8_t *key) noexcept { return visit_string(iter, key, true); } -simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator &iter) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::increment_count(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 return SUCCESS; } -simdjson_inline tape_builder::tape_builder(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} +template +simdjson_inline tape_builder_impl::tape_builder_impl(dom::document &doc) noexcept : tape{doc.tape.get()}, current_string_buf_loc{doc.string_buf.get()} {} -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_string(json_iterator &iter, const uint8_t *value, bool key) noexcept { iter.log_value(key ? "key" : "string"); uint8_t *dst = on_start_string(iter); - dst = stringparsing::parse_string(value+1, dst, false); // We do not allow replacement when the escape characters are invalid. + // We do not allow replacement when the escape characters are invalid. + // UNPADDED is a compile-time constant chosen once per document by + // tape_builder::parse_document, so the padded build instantiates only the + // plain parse_string call below -- no runtime branch and no flag load. + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + dst = stringparsing::parse_string_safe(value+1, dst, false, iter.buf + iter.dom_parser.len); + } else { + dst = stringparsing::parse_string(value+1, dst, false); + } if (dst == nullptr) { iter.log_error("Invalid escape in string"); return STRING_ERROR; @@ -64868,27 +80251,48 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_ return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_string(json_iterator &iter, const uint8_t *value) noexcept { return visit_string(iter, value); } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_number(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("number"); - error_code err = numberparsing::parse_number(value, tape); + const uint8_t *num = value; + std::unique_ptr copy{}; // keeps a padded copy of the tail alive when used + SIMDJSON_IF_CONSTEXPR (UNPADDED) { + // numberparsing reads ahead in 8-byte blocks for floats + // (is_made_of_eight_digits_fast reads up to 7 bytes past the digits), so a + // number whose digits reach the final bytes of an unpadded buffer would read + // past it. *(next_structural) is the offset of the token following this + // number, hence an upper bound on where the digits end; when that is within + // SIMDJSON_PADDING of the end we parse from a space-padded copy of the tail + // (mirroring visit_root_number). This fires only for numbers near the end. + if (simdjson_unlikely(*(iter.next_structural) + SIMDJSON_PADDING > iter.dom_parser.len)) { + const size_t rl = iter.remaining_len(); // bytes from `value` to the end of the document + copy.reset(new (std::nothrow) uint8_t[rl + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, rl); + std::memset(copy.get() + rl, ' ', SIMDJSON_PADDING); + num = copy.get(); + } + } + error_code err = numberparsing::parse_number(num, tape); if (simdjson_unlikely(err == BIGINT_ERROR && iter.dom_parser._number_as_string)) { // Write big integer to string buffer using the same format as strings. // Scan digits the same way parse_number does (skip optional '-', then digits). - const uint8_t *p = value; + const uint8_t *p = num; if (*p == '-') p++; while (numberparsing::is_digit(*p)) p++; // The digit run must be terminated by a structural or whitespace character; otherwise the // token is malformed (e.g. "123456789123456789123x"). if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - size_t len = size_t(p - value); + size_t len = size_t(p - num); tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::BIGINT); uint8_t *dst = current_string_buf_loc + sizeof(uint32_t); - memcpy(dst, value, len); + memcpy(dst, num, len); dst += len; on_end_string(dst); return SUCCESS; @@ -64896,7 +80300,8 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_ return err; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_number(json_iterator &iter, const uint8_t *value) noexcept { // // We need to make a copy to make sure that the string is space terminated. // This is not about padding the input, which should already padded up @@ -64910,76 +80315,142 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number( // practice unless you are in the strange scenario where you have many JSON // documents made of single atoms. // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + // In a stream, the input goes on with other documents: copy up to the next + // structural only, not to the end of the batch. + const size_t len = (std::min)(iter.remaining_len(), size_t(*iter.next_structural) - size_t(*(iter.next_structural - 1))); + std::unique_ptrcopy(new (std::nothrow) uint8_t[len + SIMDJSON_PADDING]); if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + std::memcpy(copy.get(), value, len); + std::memset(copy.get() + len, ' ', SIMDJSON_PADDING); error_code error = visit_number(iter, copy.get()); return error; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + // The non-length-aware validator reads a fixed 5 bytes; a malformed/truncated + // token at the very end of an unpadded buffer would over-read. Use the + // length-aware form there (the root variant already does this). + const bool ok = UNPADDED ? atomparsing::is_valid_true_atom(value, iter.remaining_len()) + : atomparsing::is_valid_true_atom(value); + if (!ok) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_true_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("true"); if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } tape.append(0, internal::tape_type::TRUE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_false_atom(value, iter.remaining_len()) + : atomparsing::is_valid_false_atom(value); + if (!ok) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_false_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("false"); if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } tape.append(0, internal::tape_type::FALSE_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + const bool ok = UNPADDED ? atomparsing::is_valid_null_atom(value, iter.remaining_len()) + : atomparsing::is_valid_null_atom(value); + if (!ok) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } -simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_null_atom(json_iterator &iter, const uint8_t *value) noexcept { iter.log_value("null"); if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } tape.append(0, internal::tape_type::NULL_VALUE); return SUCCESS; } +#if SIMDJSON_ENABLE_NAN_INF +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + // For unpadded input use the length-aware validator so the 'infinity'-style + // 8-byte compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_nan_atom(value, iter.remaining_len()) + : atomparsing::is_valid_nan_atom(value); + if (!ok) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_nan_atom(json_iterator &iter, const uint8_t *value, error_code errc) noexcept { + iter.log_value("nan"); + if (!atomparsing::is_valid_nan_atom(value, iter.remaining_len())) { return errc; } + tape.append_double(std::numeric_limits::quiet_NaN()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure. + // For unpadded input use the length-aware validator so the 'infinity' 8-byte + // compare cannot read past the buffer on a malformed token at the end. + const bool ok = UNPADDED ? atomparsing::is_valid_inf_atom(value, iter.remaining_len()) + : atomparsing::is_valid_inf_atom(value); + if (!ok) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} + +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::visit_root_inf_atom(json_iterator &iter, const uint8_t *value) noexcept { + iter.log_value("inf"); + // Because 'inf' is an extension, non a canonical atom, a tape error should be returned on failure + if (!atomparsing::is_valid_inf_atom(value, iter.remaining_len())) { return TAPE_ERROR; } + tape.append_double(std::numeric_limits::infinity()); + return SUCCESS; +} +#endif // SIMDJSON_ENABLE_NAN_INF + // private: -simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator &iter) const noexcept { +template +simdjson_inline uint32_t tape_builder_impl::next_tape_index(json_iterator &iter) const noexcept { return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); } -simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::empty_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { auto start_index = next_tape_index(iter); tape.append(start_index+2, start); tape.append(start_index, end); return SUCCESS; } -simdjson_inline void tape_builder::start_container(json_iterator &iter) noexcept { +template +simdjson_inline void tape_builder_impl::start_container(json_iterator &iter) noexcept { iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); iter.dom_parser.open_containers[iter.depth].count = 0; tape.skip(); // We don't actually *write* the start element until the end. } -simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { +template +simdjson_warn_unused simdjson_inline error_code tape_builder_impl::end_container(json_iterator &iter, internal::tape_type start, internal::tape_type end) noexcept { // Write the ending tape element, pointing at the start location const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; tape.append(start_tape_index, end); @@ -64992,13 +80463,15 @@ simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json return SUCCESS; } -simdjson_inline uint8_t *tape_builder::on_start_string(json_iterator &iter) noexcept { +template +simdjson_inline uint8_t *tape_builder_impl::on_start_string(json_iterator &iter) noexcept { // we advance the point, accounting for the fact that we have a NULL termination tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); return current_string_buf_loc + sizeof(uint32_t); } -simdjson_inline void tape_builder::on_end_string(uint8_t *dst) noexcept { +template +simdjson_inline void tape_builder_impl::on_end_string(uint8_t *dst) noexcept { uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); // TODO check for overflow in case someone has a crazy string (>=4GB?) // But only add the overflow check when the document itself exceeds 4GB @@ -65229,14 +80702,14 @@ simdjson_warn_unused simdjson_inline error_code scan() { // Primitive or invalid character (invalid characters will be checked in stage 2) } else { // Anything else, add the structural and go until we find the next one. - // We also stop on '"' so that an unclosed string still reaches - // validate_string(); a quote swallowed by the run would hide it. A - // quote cannot occur inside a valid primitive. We deliberately do not - // stop on every ESC_ASCII character: that also covers a backslash and the - // control characters, and ending the run there makes the fallback - // disagree with the SIMD kernels. + // We also stop on RS (0x1E) so that RFC 7464 json_sequence inputs + // like `\x1e"a"\x1e"b"` produce a separate structural for each RS + // rather than being absorbed into a single primitive run, and on '"' + // so that an unclosed string still reaches validate_string(). Neither + // can occur inside a valid primitive. add_structural(); - while (idx+1= 202402L) // update when the standard is finalized +// While C++26 is a working draft, compilers report 202400L in C++26 mode +// (both GCC 16 and Clang 21 do). Update when the standard is finalized. +#if !defined(SIMDJSON_CPLUSPLUS26) && (SIMDJSON_CPLUSPLUS >= 202400L) #define SIMDJSON_CPLUSPLUS26 1 #endif @@ -118,14 +120,48 @@ #endif #endif -// The current specification is unclear on how we detect -// static reflection, both __cpp_lib_reflection and -// __cpp_impl_reflection are proposed in the draft specification. -// For now, we disable static reflect by default. It must be -// specified at compiler time. +// Static reflection. +// +// The reflection-based APIs (simdjson::to, document::get, the builder, +// compile-time JSON, annotations) need considerably more than the reflection +// operator. We turn them on only when the compiler advertises all of: +// +// P2996 reflection (^^, splicers, ) __cpp_impl_reflection, +// __cpp_lib_reflection +// P1306 expansion statements (template for) __cpp_expansion_statements +// P3491 std::define_static_string / _array __cpp_lib_define_static +// +// Two further features we rely on have, as of this writing, no feature-test +// macro of their own, so they cannot be checked directly: +// +// P3394 annotations ([[=x]], std::meta::annotations_of) -- used for +// the annotations of simdjson/annotations.h (rename, skip, ...). +// P3289 consteval blocks (consteval { ... }) -- used by compile_time_json. +// +// Every implementation that defines the four macros above also implements +// those two, so requiring the four is sufficient in practice. If that ever +// stops being true, define SIMDJSON_STATIC_REFLECTION=0 to opt out. +// +// SIMDJSON_STATIC_REFLECTION may always be defined by the user (or by the +// build system) to 0 or 1 to override the detection. +// +// Note that C++26 mode alone is not enough: GCC 16 requires -freflection, +// and only then does it define __cpp_impl_reflection. #ifndef SIMDJSON_STATIC_REFLECTION -#define SIMDJSON_STATIC_REFLECTION 0 // disabled by default. +#if defined(SIMDJSON_CPLUSPLUS26) && \ + defined(__cpp_impl_reflection) && __cpp_impl_reflection >= 202506L && \ + defined(__cpp_lib_reflection) && __cpp_lib_reflection >= 202506L && \ + defined(__cpp_expansion_statements) && \ + __cpp_expansion_statements >= 202506L && \ + defined(__cpp_lib_define_static) && __cpp_lib_define_static >= 202506L +// __cpp_lib_reflection is the feature-test macro for , so there is no +// need for a separate __has_include check (which would have to be guarded for +// compilers that lack __has_include). +#define SIMDJSON_STATIC_REFLECTION 1 +#else +#define SIMDJSON_STATIC_REFLECTION 0 #endif +#endif // SIMDJSON_STATIC_REFLECTION #if defined(__apple_build_version__) #if __apple_build_version__ < 14000000 @@ -158,6 +194,47 @@ #define SIMDJSON_SUPPORTS_DESERIALIZATION 0 #endif +// The C++20 char8_t type (and std::u8string/std::u8string_view) is available. +// Because all strings that simdjson produces are valid UTF-8, we can offer +// char8_t variants of our string accessors when this macro is set. +#if !defined(SIMDJSON_SUPPORTS_CHAR8_T) +#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L +#define SIMDJSON_SUPPORTS_CHAR8_T 1 +#else +#define SIMDJSON_SUPPORTS_CHAR8_T 0 +#endif +#endif // !defined(SIMDJSON_SUPPORTS_CHAR8_T) + +// The C++23 fixed-width floating-point types std::float32_t and std::float64_t +// () are available. They are optional even in C++23: a compiler that +// provides them predefines __STDCPP_FLOAT32_T__ and __STDCPP_FLOAT64_T__. +// When these macros are set, we offer get_float32() and get_float64(). +#if !defined(SIMDJSON_SUPPORTS_FLOAT32_T) +#if defined(__STDCPP_FLOAT32_T__) && defined(__has_include) +#if __has_include() +#define SIMDJSON_SUPPORTS_FLOAT32_T 1 +#endif +#endif +#ifndef SIMDJSON_SUPPORTS_FLOAT32_T +#define SIMDJSON_SUPPORTS_FLOAT32_T 0 +#endif +#endif // !defined(SIMDJSON_SUPPORTS_FLOAT32_T) + +#if !defined(SIMDJSON_SUPPORTS_FLOAT64_T) +#if defined(__STDCPP_FLOAT64_T__) && defined(__has_include) +#if __has_include() +#define SIMDJSON_SUPPORTS_FLOAT64_T 1 +#endif +#endif +#ifndef SIMDJSON_SUPPORTS_FLOAT64_T +#define SIMDJSON_SUPPORTS_FLOAT64_T 0 +#endif +#endif // !defined(SIMDJSON_SUPPORTS_FLOAT64_T) + +#if SIMDJSON_SUPPORTS_FLOAT32_T || SIMDJSON_SUPPORTS_FLOAT64_T +#include +#endif + #if !defined(SIMDJSON_CONSTEVAL) #if defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L @@ -166,6 +243,18 @@ #define SIMDJSON_CONSTEVAL 0 #endif // defined(__cpp_consteval) && __cpp_consteval >= 201811L && defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L #endif // !defined(SIMDJSON_CONSTEVAL) + +// SIMDJSON_CONSTEXPR_STRING is 'constexpr' when the standard library supports +// constexpr std::string (e.g., libstdc++ 12 or better), and empty otherwise. It +// lets functions that build a std::string be constant expressions when possible +// while still compiling against older standard libraries. +#if !defined(SIMDJSON_CONSTEXPR_STRING) +#if defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L +#define SIMDJSON_CONSTEXPR_STRING constexpr +#else +#define SIMDJSON_CONSTEXPR_STRING +#endif // defined(__cpp_lib_constexpr_string) && __cpp_lib_constexpr_string >= 201907L +#endif // !defined(SIMDJSON_CONSTEXPR_STRING) #endif // SIMDJSON_COMPILER_CHECK_H /* end file simdjson/compiler_check.h */ /* including simdjson/portability.h: #include "simdjson/portability.h" */ @@ -457,16 +546,86 @@ using std::size_t; #endif #endif +#ifndef SIMDJSON_HAS_UNISTD_H +#if defined(__unix__) || defined(__APPLE__) || defined(__linux__) +#define SIMDJSON_HAS_UNISTD_H 1 +#else +#define SIMDJSON_HAS_UNISTD_H 0 +#endif +#endif + +// padded_memory_map availability. +// +// On POSIX platforms the class is always available: the implementation uses +// `mmap` (and a trailing anonymous page for padding) from . +// +// On Windows the class is disabled by default and must be explicitly +// opted into by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. Enabling +// it requires: +// 1. `` has been included *before* `` (so that +// this header can see the Win32 types and the `_WINDOWS_` include +// guard), +// 2. the compilation targets Windows 10, version 1803 or later +// (i.e. `NTDDI_VERSION >= NTDDI_WIN10_RS4`, `0x0A000005`). This is +// required because the implementation relies on the modern memory +// APIs introduced with that version (`CreateFileMapping2` / +// `MapViewOfFile3`), +// 3. the link step pulls in an import library that exports those APIs, +// typically `onecore.lib` (or `mincore.lib`). +// +// The `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS` CMake option arranges (1)-(3) +// automatically when building simdjson with its own CMake. Consumers using +// simdjson as a pre-built library are responsible for setting the macro, +// the Windows version macros, and the link library themselves. +// +// If the opt-in conditions are not met on Windows, `padded_memory_map` +// simply does not exist -- any attempt to use it fails at compile time +// with an "unknown identifier" diagnostic rather than silently degrading. +// +// The SIMDJSON_HAS_PADDED_MEMORY_MAP macro reflects whether the class is +// available in the current translation unit. Users may test this macro to +// conditionally compile code that depends on padded_memory_map. +#ifndef SIMDJSON_HAS_PADDED_MEMORY_MAP + #if defined(__unix__) || defined(__APPLE__) || defined(__linux__) + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 + #elif defined(_WINDOWS_) && defined(SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS) && SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 1 + #else + #define SIMDJSON_HAS_PADDED_MEMORY_MAP 0 + #endif +#endif #endif // SIMDJSON_PORTABILITY_H /* end file simdjson/portability.h */ +#include namespace simdjson { namespace internal { +/** + * @private + * Scratch capacity that every caller of to_chars must provide. + * + * The emitted decimal is at most ~24 characters, but dragonbox() and + * format_buffer() intentionally write past the logical end with fixed-size + * 16/17-byte memcpy/memset operations so the compiler can inline them (no + * libc mem* dispatch with size-class branches). The extra bytes are required + * for safety of those over-writes; do not shrink this below 40. + * See src/to_chars.cpp and #2805. + */ +// Use an unscoped enum (not static constexpr / inline constexpr): +// - C++11 targets (readme_examples11, quickstart11, ...) still include this header +// - a static constexpr in the amalgamated simdjson.cpp TU is unused there +// (only callers in headers use it) and trips -Wunused-const-variable -Werror +enum : size_t { to_chars_buffer_size = 40 }; /** * @private * Our own implementation of the C++17 to_chars function. * Defined in src/to_chars + * + * @note The buffer starting at first must have at least to_chars_buffer_size + * bytes of writable storage (see to_chars_buffer_size). + * @note The input number must be finite (NaN/Inf are not supported). + * @note The result is NOT null-terminated. */ char *to_chars(char *first, const char *last, double value); /** @@ -476,6 +635,12 @@ char *to_chars(char *first, const char *last, double value); */ double from_chars(const char *first) noexcept; double from_chars(const char *first, const char* end) noexcept; +/** + * @private + * Same as from_chars, but produces a correctly rounded binary32 (float) value. + * Defined in src/from_chars + */ +float from_chars_float(const char *first) noexcept; } #ifndef SIMDJSON_EXCEPTIONS @@ -486,6 +651,10 @@ double from_chars(const char *first, const char* end) noexcept; #endif #endif +#ifndef SIMDJSON_ENABLE_NAN_INF +#define SIMDJSON_ENABLE_NAN_INF 0 +#endif + } // namespace simdjson #if defined(__GNUC__) @@ -501,16 +670,14 @@ double from_chars(const char *first, const char* end) noexcept; // Align to N-byte boundary #define SIMDJSON_ROUNDUP_N(a, n) (((a) + ((n)-1)) & ~((n)-1)) -#define SIMDJSON_ROUNDDOWN_N(a, n) ((a) & ~((n)-1)) - -#define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) #if SIMDJSON_REGULAR_VISUAL_STUDIO // We could use [[deprecated]] but it requires C++14 #define simdjson_deprecated __declspec(deprecated) #define simdjson_really_inline __forceinline - #define simdjson_never_inline __declspec(noinline) + #define simdjson_never_inline inline __declspec(noinline) + #define simdjson_really_flatten [[msvc::flatten]] #define simdjson_unused #define simdjson_warn_unused @@ -551,6 +718,7 @@ double from_chars(const char *first, const char* end) noexcept; #define simdjson_really_inline inline __attribute__((always_inline)) #define simdjson_never_inline inline __attribute__((noinline)) + #define simdjson_really_flatten [[gnu::flatten]] #define simdjson_unused __attribute__((unused)) #define simdjson_warn_unused __attribute__((warn_unused_result)) @@ -627,6 +795,15 @@ double from_chars(const char *first, const char* end) noexcept; #define simdjson_inline simdjson_really_inline #endif +#if defined(simdjson_flatten) + // Prefer the user's definition of simdjson_flatten; don't define it ourselves. +#elif (defined(__GNUC__) && !defined(__OPTIMIZE__)) || (defined(_DEBUG) && _MSC_VER ) + // Flattening can lead to significant code bloat and high compile times. Don't use it for unoptimized builds. + #define simdjson_flatten +#else + #define simdjson_flatten simdjson_really_flatten +#endif + #if SIMDJSON_VISUAL_STUDIO /** * Windows users need to do some extra work when building @@ -2538,22 +2715,22 @@ namespace std { #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "4.6.11" +#define SIMDJSON_VERSION "5.0.2" namespace simdjson { enum { /** * The major version (MAJOR.minor.revision) of simdjson being used. */ - SIMDJSON_VERSION_MAJOR = 4, + SIMDJSON_VERSION_MAJOR = 5, /** * The minor version (major.MINOR.revision) of simdjson being used. */ - SIMDJSON_VERSION_MINOR = 6, + SIMDJSON_VERSION_MINOR = 0, /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 11 + SIMDJSON_VERSION_REVISION = 2 }; } // namespace simdjson @@ -2625,6 +2802,7 @@ enum error_code { OUT_OF_BOUNDS, ///< Attempted to access location outside of document. TRAILING_CONTENT, ///< Unexpected trailing content in the JSON input OUT_OF_CAPACITY, ///< The capacity was exceeded, we cannot allocate enough memory. + UNKNOWN_FIELD, ///< JSON field does not map to any member of the target (see simdjson::deny_unknown_fields) NUM_ERROR_CODES ///< Placeholder for end of error code list. }; @@ -2987,6 +3165,7 @@ inline const std::string error_message(int error) noexcept; #if SIMDJSON_SUPPORTS_CONCEPTS #include +#include #include namespace simdjson { @@ -3028,6 +3207,19 @@ concept constructible_from_string_view = std::is_constructible_v && std::is_default_constructible_v; +#if SIMDJSON_SUPPORTS_CHAR8_T +/** + * A C++20 char8_t string type such as std::u8string. Such types cannot be built + * from a std::string_view (the character types differ), so they need their own + * deserialization path, going through the u8 string accessors. + */ +template +concept constructible_from_u8string_view = std::is_constructible_v + && !std::is_same_v + && !std::is_constructible_v + && std::is_default_constructible_v; +#endif // SIMDJSON_SUPPORTS_CHAR8_T + template concept string_view_keyed_map = string_view_like && requires(std::remove_cvref_t& m, typename M::key_type sv, typename M::mapped_type v) { @@ -3122,9 +3314,15 @@ concept string_like = // Concept that checks if a type is a container but not a string (because // strings handling must be handled differently) // Now uses iterator-based approach for broader container support +// +// Optional types are excluded on purpose. Since C++26 (P3168), std::optional +// is itself a range, so without the exclusion an std::optional would match +// both this concept and optional_type, making the container and the optional +// overloads of atom()/append() ambiguous. See issue 2827. template concept container_but_not_string = - std::ranges::input_range && !string_like && !concepts::string_view_keyed_map; + std::ranges::input_range && !string_like && !concepts::string_view_keyed_map + && !concepts::optional_type; @@ -3231,6 +3429,11 @@ struct fixed_string { data[i] = str[i]; } } + constexpr fixed_string(const unsigned char (&str)[N]) { + for (std::size_t i = 0; i < N; ++i) { + data[i] = static_cast(str[i]); + } + } char data[N]; constexpr std::string_view view() const { return {data, N - 1}; } constexpr size_t size() const { return N ; } @@ -3262,6 +3465,11 @@ struct string_constant { #endif // SIMDJSON_CONSTEVALUTIL_H /* end file simdjson/constevalutil.h */ +#if SIMDJSON_SUPPORTS_CHAR8_T +#include +#include +#endif + /** * @brief The top level simdjson namespace, containing everything the library provides. */ @@ -3271,6 +3479,10 @@ SIMDJSON_PUSH_DISABLE_UNUSED_WARNINGS /** The maximum document size supported by simdjson. */ constexpr size_t SIMDJSON_MAXSIZE_BYTES = 0xFFFFFFFF; +/** The maximum depth of nested objects and arrays supported by simdjson. + A depth of SIMDJSON_MAXSIZE_BYTES/2 is not reasonable and would be + adversarial, but it serves as an upper bound for validation purposes. */ +constexpr size_t SIMDJSON_MAX_DEPTH = SIMDJSON_MAXSIZE_BYTES/2; /** * The amount of padding needed in a buffer to parse JSON. @@ -3296,6 +3508,30 @@ struct padded_string; class padded_string_view; enum class stage1_mode; +/** + * Stream format for parse_many/iterate_many. + */ +enum class stream_format { + whitespace_delimited, ///< Whitespace-delimited JSON documents (default, includes NDJSON/JSONL) + json_sequence, ///< RFC 7464 JSON text sequences (RS-delimited) + comma_delimited, ///< Comma-separated JSON documents (e.g., `{...},{...},{...}`) + comma_delimited_array,///< A single JSON array whose elements are iterated as + ///< comma-separated documents (e.g., `[{...},{...},{...}]`). + ///< The parser strips the outer `[` / `]` plus any + ///< surrounding JSON whitespace (space, tab, LF, CR) + ///< and then behaves like `comma_delimited` over the + ///< remaining bytes. + newline_delimited ///< NDJSON/JSON Lines where each document occupies exactly + ///< one line: documents are separated by line feeds and no + ///< document contains a raw line feed. Same inputs as + ///< `whitespace_delimited`, but the stronger guarantee lets + ///< the parser find the end of a document without walking + ///< it. On ondemand `iterate_many`, an unread remainder may + ///< be skipped by jumping to the next line feed without + ///< structure-validating that remainder. Use + ///< `whitespace_delimited` if unsure. +}; + namespace internal { template @@ -3306,6 +3542,52 @@ class tape_ref; struct value128; enum class tape_type; +#if SIMDJSON_SUPPORTS_CHAR8_T +/** + * Reinterpret a UTF-8 string as a C++20 std::u8string_view. No byte is copied + * or modified: char8_t and char have the same size, representation and + * alignment. Every string that simdjson produces is valid UTF-8, so this is a + * lossless view over the very same memory. + * @private + */ +simdjson_inline std::u8string_view as_u8string_view(std::string_view v) noexcept { + return std::u8string_view(reinterpret_cast(v.data()), v.size()); +} +#endif // SIMDJSON_SUPPORTS_CHAR8_T + +/** + * Assign a UTF-8 string to a string-like receiver. The general case simply + * assigns the std::string_view: it covers std::string and any user type that + * can be assigned from a std::string_view. + * @private + */ +template +simdjson_inline void assign_utf8(string_type &receiver, std::string_view content) noexcept { + receiver = content; +} + +#if SIMDJSON_SUPPORTS_CHAR8_T +/** + * Assign a UTF-8 string to a char8_t-based string (e.g., std::u8string). This + * overload is more specialized than the general one, so overload resolution + * prefers it whenever the receiver holds char8_t. + * @private + */ +template +simdjson_inline void assign_utf8(std::basic_string &receiver, std::string_view content) noexcept { + receiver.assign(reinterpret_cast(content.data()), content.size()); +} + +/** + * Assign a UTF-8 string to a char8_t-based string view (e.g., std::u8string_view). + * @private + */ +template +simdjson_inline void assign_utf8(std::basic_string_view &receiver, std::string_view content) noexcept { + receiver = std::basic_string_view(reinterpret_cast(content.data()), content.size()); +} +#endif // SIMDJSON_SUPPORTS_CHAR8_T + } // namespace internal } // namespace simdjson @@ -3599,7 +3881,12 @@ class document; * 3) The stream_final mode allows us to truncate final * unterminated strings. It is useful in conjunction with streaming_partial. */ -enum class stage1_mode { regular, streaming_partial, streaming_final}; +enum class stage1_mode { + regular, + streaming_partial, streaming_final, + json_sequence_partial, json_sequence_final, + comma_delimited_partial, comma_delimited_final +}; /** * Returns true if mode == streaming_partial or mode == streaming_final @@ -3611,7 +3898,6 @@ inline bool is_streaming(stage1_mode mode) { // return (mode == stage1_mode::streaming_partial || mode == stage1_mode::streaming_final); } - namespace internal { @@ -3796,6 +4082,16 @@ class dom_parser_implementation { /** Whether to store big integers as strings instead of returning BIGINT_ERROR */ bool _number_as_string{false}; + /** + * Whether the input buffer passed to parse() is *not* padded to len + + * SIMDJSON_PADDING bytes. When true, stage 2 string parsing avoids reading + * past buf+len (it finishes the final, near-the-end bytes from a small padded + * scratch buffer). This is set only by the no-padding DOM parse entry points + * (dom::parser::parse_unpadded); the default padded fast path leaves it false + * and is unaffected. + */ + bool _unpadded{false}; + protected: // Declaring these so that subclasses can use them to implement their constructors. @@ -4350,11 +4646,26 @@ inline std::ostream& operator<<(std::ostream& out, const padded_string& s) { ret inline std::ostream& operator<<(std::ostream& out, simdjson_result &s) noexcept(false) { return out << s.value(); } #endif - -#ifndef _WIN32 +#if SIMDJSON_HAS_PADDED_MEMORY_MAP /** * A class representing a memory-mapped file with padding. - * It is only available on non-Windows platforms, as Windows has different APIs for memory mapping. + * + * On POSIX systems (Linux, macOS, BSD, ...), this uses `mmap` to map the file + * contents directly into memory, which is efficient for large files (no copy). + * + * On Windows, this class is disabled by default and must be opted into at + * build time by defining `SIMDJSON_ENABLE_MEMORY_FILE_MAPPING_ON_WINDOWS=1`. When + * enabled, `` must also be included before `` and + * the compilation must target Windows 10, version 1803 or later. The + * Windows implementation uses the modern memory APIs (`VirtualAlloc2`, + * `CreateFileMapping2`, `MapViewOfFile3`) with the placeholder virtual + * memory mechanism to always achieve true zero-copy mapping with + * contiguous zero-filled padding. + * + * Either way, the resulting `padded_string_view` carries at least + * `SIMDJSON_PADDING` bytes of accessible zero-filled padding after the file + * content, so it can be consumed directly by the simdjson parsers (including + * `parse_many` / `iterate_many`). */ class padded_memory_map { public: @@ -4362,9 +4673,11 @@ class padded_memory_map { * Create a new padded memory map for the given file. * After creating the memory map, you can call view() to get a padded_string_view of the file content. * The memory map will be automatically released when the padded_memory_map instance is destroyed. - * Note that the file content is not copied, so this is efficient for large files. However, - * the file must remain unchanged while the memory map is in use. In case of error (e.g., file not found, - * permission denied, etc.), the memory map will be invalid and view() will return an empty view. + * On POSIX systems, the file content is not copied, so this is efficient for large files. + * On Windows, the file is mapped into memory via `MapViewOfFile3` (zero-copy). + * In all cases, the file must remain unchanged while the memory map is in use. + * In case of error (e.g., file not found, permission denied, etc.), the memory map will be + * invalid and view() will return an empty view. * You can check if the memory map is valid by calling is_valid() before using view(). * * @param filename the path to the file to memory-map. @@ -4401,8 +4714,14 @@ class padded_memory_map { padded_memory_map &operator=(const padded_memory_map &) = delete; const char *data{nullptr}; size_t size{0}; +#ifdef _WIN32 + // When the file ends near an allocation-granularity boundary, we use the + // placeholder API to append zero-filled padding pages. This pointer tracks + // that region so the destructor can release it with VirtualFree. + void *padding_view_{nullptr}; +#endif }; -#endif // _WIN32 +#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP @@ -4476,6 +4795,9 @@ simdjson_warn_unused error_code minify(const char *buf, size_t len, char *dst, s #include #include #include +#if SIMDJSON_CPLUSPLUS17 +#include +#endif namespace simdjson { @@ -4540,6 +4862,68 @@ class padded_string_view : public std::string_view { }; // padded_string_view +/** + * Get the system's memory page size. By default, we return + * 4096 bytes, which is the most common page size. On systems + * where the page size is not a multiple of 4096 bytes, and not + * a unix-like system, nor Windows, this function may return an + * incorrect value. + * + * @return The page size in bytes. + */ +inline uint32_t get_page_size() noexcept; + +#if SIMDJSON_CPLUSPLUS17 +/** + * A padded_input is a wrapper around either a padded_string_view or a padded_string. + * It will automatically pad a string_view if it does not have sufficient padding + * up to the end of the memory page. Note that a requirement for this method to + * make sense is to be on a system with a page size of at least 4096 (which is + * universal except on some embedded systems). + */ +struct padded_input { + /** + * Construct a padded_input from a string_view. If the string_view does not have sufficient padding, + * the data will be copied into a padded_string and the padded_string_view will point to the + * padded_string's data. Otherwise, the padded_string_view will point to the original string_view's data. + */ + inline explicit padded_input(std::string_view sv); + /** + * Construct a padded_input from a C-style string (length specified). If the string does not have sufficient padding, + * the data will be copied into a padded_string and the padded_string_view will point to the + * padded_string's data. Otherwise, the padded_string_view will point to the original string's data. + */ + inline explicit padded_input(const char *data, size_t length); + /** + * Construct a padded_input from a std::string. If the string does not have sufficient padding + * (considering its capacity), the data will be copied into a padded_string and the padded_string_view + * will point to the padded_string's data. Otherwise, the padded_string_view will point to the + * original string's data. + */ + inline explicit padded_input(const std::string &s); + + /** + * Check if the padded_input is a view. + * + * @return true if the padded_input is a view, false otherwise. + */ + inline bool is_view() const noexcept; + + /** + * Convert the padded_input to a padded_string_view. + * + * @return The padded_string_view. + */ + inline operator simdjson::padded_string_view() const noexcept; + +private: + std::variant storage; + // whether we cross a page boundary and need to allocate a new padded string. + static inline bool needs_allocation(const char* buf, size_t len, size_t padding = SIMDJSON_PADDING) noexcept; +}; + +#endif // SIMDJSON_CPLUSPLUS17 + #if SIMDJSON_EXCEPTIONS /** * Send padded_string instance to an output stream. @@ -4572,6 +4956,31 @@ inline padded_string_view pad(std::string& s) noexcept; * @return The padded string. */ inline padded_string_view pad_with_reserve(std::string& s) noexcept; + +/** + * Return the index-th document-aligned slice of a delimited stream. + * + * The input is divided into blocks of block_size bytes and each boundary is + * moved forward to just past the next delimiter, so a document is never split. + * The delimiter must not occur inside a document: a line feed for NDJSON, a + * record separator (0x1E) for RFC 7464. + * + * Slices are contiguous and non-overlapping, and each may be parsed + * independently, so callers can process them on as many threads as they like. + * + * Iterate while index * block_size < data.size(). A slice is empty when its + * block falls entirely inside one document, which happens only if that document + * is longer than block_size; skip it and continue. With block_size larger than + * the longest document, no slice is ever empty. + * + * @param data The padded input. + * @param delimiter The byte that separates documents. + * @param block_size The nominal slice size, before snapping. + * @param index Which slice to return, counting from zero. + */ +inline padded_string_view slice_at(padded_string_view data, char delimiter, + size_t block_size, size_t index) noexcept; + } // namespace simdjson #endif // SIMDJSON_PADDED_STRING_VIEW_H @@ -4588,6 +4997,15 @@ inline padded_string_view pad_with_reserve(std::string& s) noexcept; #include /* memcmp */ +// for page size computation. +#if SIMDJSON_HAS_UNISTD_H + #include + #if defined(__APPLE__) + #include + #endif +#endif + + namespace simdjson { inline padded_string_view::padded_string_view(const char* s, size_t len, size_t capacity) noexcept @@ -4677,7 +5095,102 @@ inline padded_string_view pad_with_reserve(std::string& s) noexcept { return padded_string_view(s.data(), s.size(), s.capacity()); } +inline uint32_t get_page_size() noexcept { +#if defined(_WINDOWS_) // if and only if someone loaded Windows.h, we can get the page size from there. +// Otherwise, we assume 4096. + static const uint32_t cached = []() -> uint32_t { + SYSTEM_INFO si; + GetSystemInfo(&si); + return static_cast(si.dwPageSize); + }(); + return cached; +#elif SIMDJSON_HAS_UNISTD_H + static const uint32_t cached = []() -> uint32_t { + long page_size = sysconf(_SC_PAGESIZE); + if (page_size > 0) { + return static_cast(page_size); + } + return 4096; // fallback + }(); + return cached; +#else + return 4096; // fallback +#endif +} +#if SIMDJSON_CPLUSPLUS17 + +inline padded_input::padded_input(std::string_view sv) + : storage(simdjson::padded_string_view{}) { + if (needs_allocation(sv.data(), sv.size())) { + storage = simdjson::padded_string(sv); + } else { + storage = simdjson::padded_string_view( + sv.data(), sv.size(), sv.size() + simdjson::SIMDJSON_PADDING); + } +} + +inline padded_input::padded_input(const char *data, size_t length) + : storage(simdjson::padded_string_view{}) { + if (needs_allocation(data, length)) { + storage = simdjson::padded_string(data, length); + } else { + storage = simdjson::padded_string_view( + data, length, length + simdjson::SIMDJSON_PADDING); + } +} +inline padded_input::padded_input(const std::string &s) + : storage(simdjson::padded_string_view{}) { + const size_t len = s.size(); + const size_t cap = s.capacity(); + // Here we have the string content from data() to data() + size(), + // but the memory is accessible from data() to data() + capacity(). + const size_t needed_padding = (cap - len) < simdjson::SIMDJSON_PADDING + ? simdjson::SIMDJSON_PADDING - (cap - len) : 0; + if (needed_padding > 0 && needs_allocation(s.data(), cap, needed_padding)) { + storage = simdjson::padded_string(s); + } else { + storage = simdjson::padded_string_view( + s.data(), len, len + simdjson::SIMDJSON_PADDING); + } +} + +inline bool padded_input::is_view() const noexcept { + return std::holds_alternative(storage); +} + +inline padded_input::operator simdjson::padded_string_view() const noexcept { + return std::visit([](const auto& p) -> simdjson::padded_string_view { + return p; + }, storage); +} + +inline bool padded_input::needs_allocation(const char* buf, size_t len, size_t padding) noexcept { + if(len == 0) { return false; } + const auto page_size = get_page_size(); + return ((reinterpret_cast(buf + len - 1) % page_size) + + padding >= static_cast(page_size)); +} +#endif // SIMDJSON_CPLUSPLUS17 + +inline padded_string_view slice_at(padded_string_view data, char delimiter, + size_t block_size, size_t index) noexcept { + if (block_size == 0 || index > data.size() / block_size) { return {}; } + const size_t raw_begin = index * block_size; + if (raw_begin >= data.size()) { return {}; } + + auto snap = [&](size_t want) -> size_t { + if (want >= data.size()) { return data.size(); } + const void *p = std::memchr(data.data() + want, delimiter, data.size() - want); + return p ? size_t(static_cast(p) - data.data()) + 1 : data.size(); + }; + + const size_t begin = (raw_begin == 0) ? 0 : snap(raw_begin); + const size_t end = snap(raw_begin + block_size); + if (begin >= end) { return {}; } + return padded_string_view(data.data() + begin, end - begin, + data.capacity() - begin); +} } // namespace simdjson @@ -4688,13 +5201,18 @@ inline padded_string_view pad_with_reserve(std::string& s) noexcept { #include #include -#ifndef _WIN32 +#if SIMDJSON_HAS_UNISTD_H #include #include #include #include #include #endif +// On Windows, `padded_memory_map` (when it is enabled) depends on types and +// functions declared in . We deliberately do NOT include that +// header here: users of simdjson who want `padded_memory_map` on Windows +// must include themselves *before* including this header. See +// padded_string.h for the detection logic. namespace simdjson { namespace internal { @@ -5064,7 +5582,9 @@ inline bool padded_string_builder::reserve(size_t additional) noexcept { } -#ifndef _WIN32 +#if SIMDJSON_HAS_PADDED_MEMORY_MAP + +#if SIMDJSON_HAS_UNISTD_H simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { int fd = open(filename, O_RDONLY); @@ -5100,7 +5620,132 @@ simdjson_inline padded_memory_map::~padded_memory_map() noexcept { munmap(const_cast(data), size + simdjson::SIMDJSON_PADDING); } } +#elif defined(_WIN32) +// Windows zero-copy implementation using placeholder virtual memory. +// +// We use the modern Windows memory APIs (VirtualAlloc2, CreateFileMapping2, +// MapViewOfFile3 -- available since Windows 10 1803) to map the file into a +// contiguous virtual address range that includes at least SIMDJSON_PADDING +// zero bytes after the file content, with no data copies. +// +// Strategy: +// 1. If rounding the file size up to the allocation granularity already +// exceeds file_size + SIMDJSON_PADDING, the OS page zero-fill provides +// the padding and we use a simple MapViewOfFile3 call. +// 2. Otherwise we reserve a contiguous placeholder region via VirtualAlloc2, +// split it at the granularity-aligned file boundary, map the file into +// the first part, and commit zero pages for the second part (padding). +simdjson_inline padded_memory_map::padded_memory_map(const char *filename) noexcept { + HANDLE file_handle = ::CreateFileA( + filename, GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (file_handle == INVALID_HANDLE_VALUE) { + return; + } + LARGE_INTEGER file_size_li; + if (!::GetFileSizeEx(file_handle, &file_size_li) || file_size_li.QuadPart < 0) { + ::CloseHandle(file_handle); + return; + } +#if SIMDJSON_IS_32BITS + if (static_cast(file_size_li.QuadPart) > + static_cast(SIZE_MAX - simdjson::SIMDJSON_PADDING)) { + ::CloseHandle(file_handle); + return; + } +#endif + size = static_cast(file_size_li.QuadPart); + if (size == 0) { + ::CloseHandle(file_handle); + return; + } + + HANDLE section = ::CreateFileMapping2( + file_handle, NULL, FILE_MAP_READ, PAGE_READONLY, + 0, 0, NULL, NULL, 0); + ::CloseHandle(file_handle); + if (section == NULL) { + return; + } + + SYSTEM_INFO si; + ::GetSystemInfo(&si); + const size_t granularity = static_cast(si.dwAllocationGranularity); + const size_t file_region = (size + granularity - 1) & ~(granularity - 1); + const size_t total_needed = size + simdjson::SIMDJSON_PADDING; + + if (file_region >= total_needed) { + // The zero-fill in the last page already covers the padding. + PVOID view = ::MapViewOfFile3( + section, ::GetCurrentProcess(), NULL, 0, 0, + 0, PAGE_READONLY, NULL, 0); + ::CloseHandle(section); + if (view != NULL) { + data = static_cast(view); + } + return; + } + + // We need extra zero pages beyond the file region. Use the placeholder API + // to get a contiguous virtual address range spanning both the file mapping + // and the zero-filled padding. + const size_t padding_region = + ((total_needed - file_region) + granularity - 1) & ~(granularity - 1); + const size_t reserve_size = file_region + padding_region; + + // Reserve a contiguous placeholder. + PVOID placeholder = ::VirtualAlloc2( + ::GetCurrentProcess(), NULL, reserve_size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, PAGE_NOACCESS, NULL, 0); + if (placeholder == NULL) { + ::CloseHandle(section); + return; + } + // Split into two placeholders at the file_region boundary. + if (!::VirtualFree(placeholder, file_region, + MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)) { + ::VirtualFree(placeholder, 0, MEM_RELEASE); + ::CloseHandle(section); + return; + } + + // Map the file into the first placeholder. + PVOID file_view = ::MapViewOfFile3( + section, ::GetCurrentProcess(), placeholder, 0, file_region, + MEM_REPLACE_PLACEHOLDER, PAGE_READONLY, NULL, 0); + ::CloseHandle(section); + if (file_view == NULL) { + ::VirtualFree(placeholder, 0, MEM_RELEASE); + ::VirtualFree(static_cast(placeholder) + file_region, + 0, MEM_RELEASE); + return; + } + + // Commit zero pages in the second placeholder (the padding). + void *pad = static_cast(placeholder) + file_region; + PVOID padding_ptr = ::VirtualAlloc2( + ::GetCurrentProcess(), pad, padding_region, + MEM_REPLACE_PLACEHOLDER | MEM_COMMIT, PAGE_READONLY, NULL, 0); + if (padding_ptr == NULL) { + ::UnmapViewOfFile(file_view); + ::VirtualFree(pad, 0, MEM_RELEASE); + return; + } + + data = static_cast(file_view); + padding_view_ = padding_ptr; +} + +simdjson_inline padded_memory_map::~padded_memory_map() noexcept { + if (data == nullptr) { return; } + ::UnmapViewOfFile(data); + if (padding_view_ != nullptr) { + ::VirtualFree(padding_view_, 0, MEM_RELEASE); + } +} +#endif // POSIX or _WIN32 simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noexcept simdjson_lifetime_bound { if(!is_valid()) { @@ -5112,7 +5757,8 @@ simdjson_inline simdjson::padded_string_view padded_memory_map::view() const noe simdjson_inline bool padded_memory_map::is_valid() const noexcept { return data != nullptr; } -#endif // _WIN32 + +#endif // SIMDJSON_HAS_PADDED_MEMORY_MAP } // namespace simdjson @@ -5222,6 +5868,9 @@ class tape_ref { simdjson_inline tape_ref() noexcept; simdjson_inline tape_ref(const dom::document *doc, size_t json_index) noexcept; inline size_t after_element() const noexcept; + // The reference must point to an element boundary inside the array whose + // opening tag is at array_start, or to that array's closing tag. + inline size_t before_element(size_t array_start) const noexcept; simdjson_inline tape_type tape_ref_type() const noexcept; simdjson_inline uint64_t tape_value() const noexcept; simdjson_inline bool is_double() const noexcept; @@ -5310,6 +5959,48 @@ class array { friend class array; }; + /** + * A forward iterator that visits the array's elements in reverse order. + * Like iterator, it returns element handles by value and does not own the + * document. No allocation or modification of the document is performed. + */ + class reverse_iterator { + public: + using value_type = element; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = value_type; + using iterator_category = std::forward_iterator_tag; + + inline reference operator*() const noexcept; + inline reverse_iterator& operator++() noexcept; + inline reverse_iterator operator++(int) noexcept; + inline bool operator==(const reverse_iterator& other) const noexcept; + inline bool operator!=(const reverse_iterator& other) const noexcept; + + reverse_iterator() noexcept = default; + reverse_iterator(const reverse_iterator&) noexcept = default; + reverse_iterator& operator=(const reverse_iterator&) noexcept = default; + private: + simdjson_inline reverse_iterator(const internal::tape_ref &tape, size_t array_start) noexcept; + internal::tape_ref tape{}; + size_t array_start{}; + friend class array; + }; + + /** + * Return the last array element, or rend() for an empty array. + * Incrementing the returned iterator moves toward the first element. + * A complete traversal takes O(n) time and O(1) additional space, where n + * is the number of immediate elements in the array. + * Finding the last or previous element can take O(n) time in the worst case when + * numeric payloads equal numeric type markers. Increments are amortized O(1) + * over a complete traversal; nested values do not increase this bound. + */ + inline reverse_iterator rbegin() const noexcept; + /** Return the reverse traversal sentinel, before the first element. */ + inline reverse_iterator rend() const noexcept; + /** * Return the first array element. * @@ -5445,6 +6136,8 @@ struct simdjson_result : public internal::simdjson_result_base parse(const char *buf) noexcept = delete; + /** + * Parse a JSON document whose buffer is **not** padded, in place and without + * copying it. + * + * *This feature is currently experimental.* + * + * The standard parse() methods require the input buffer to have at least + * SIMDJSON_PADDING extra readable bytes after the document (or they copy it + * into a padded buffer when realloc_if_needed is true). parse_unpadded() lifts + * that requirement: it parses directly from your buffer of exactly `len` bytes, + * never reading past `buf + len`, and never allocating a full padded copy. + * + * dom::parser parser; + * std::string_view json = get_json(); // no trailing padding needed + * dom::element doc = parser.parse_unpadded(json); + * + * This is the convenient way to use simdjson when you cannot (or do not want + * to) pad your input, e.g. a std::string_view into a larger buffer or a memory + * mapped file whose tail you do not control. It is generally a little slower + * than parsing a padded buffer with parse() (the very end of the document is + * handled with extra care), but it avoids the O(n) copy that + * parse(buf, len, true) performs when realloc_if_needed is true. + * + * The input is read but not modified, and it must remain valid (and the parser + * alive) for as long as you navigate the returned document, exactly like + * parse(buf, len, false). + * + * @param buf The JSON to parse. Only `len` bytes are read; no padding required. + * @param len The length of the JSON. + * @return An element pointing at the root of the document, or an error: + * - MEMALLOC if the parser does not have enough capacity and allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. + */ + inline simdjson_result parse_unpadded(const uint8_t *buf, size_t len) & noexcept; + inline simdjson_result parse_unpadded(const uint8_t *buf, size_t len) && =delete; + /** @overload parse_unpadded(const uint8_t *buf, size_t len) */ + simdjson_inline simdjson_result parse_unpadded(const char *buf, size_t len) & noexcept; + simdjson_inline simdjson_result parse_unpadded(const char *buf, size_t len) && =delete; + /** @overload parse_unpadded(const uint8_t *buf, size_t len) */ + simdjson_inline simdjson_result parse_unpadded(std::string_view s) & noexcept; + simdjson_inline simdjson_result parse_unpadded(std::string_view s) && =delete; + + /** + * Parse a non-padded JSON document into a caller-provided document instance, in + * place and without copying. This is to parse_unpadded() what + * parse_into_document() is to parse(). See parse_unpadded() for the padding and + * lifetime semantics. + * + * *This feature is currently experimental.* + * + * @param doc The document instance where the parsed data will be stored (on success). + * @param buf The JSON to parse. Only `len` bytes are read; no padding required. + * @param len The length of the JSON. + */ + inline simdjson_result parse_into_document_unpadded(document& doc, const uint8_t *buf, size_t len) & noexcept; + inline simdjson_result parse_into_document_unpadded(document& doc, const uint8_t *buf, size_t len) && =delete; + /** * Parse a JSON document into a provide document instance and return a temporary reference to it. * It is similar to the function `parse` except that instead of parsing into the internal @@ -6045,7 +6796,7 @@ class parser { * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet * spot is cache-related: small enough to fit in cache, yet big enough to * parse as many documents as possible in one tight loop. - * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * Defaults to 1MB, which has been a reasonable sweet spot in our tests. * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: * - MEMALLOC if the parser does not have enough capacity and memory allocation fails * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. @@ -6057,14 +6808,50 @@ class parser { inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result parse_many(const std::string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - inline simdjson_result parse_many(const std::string &&s, size_t batch_size) = delete;// unsafe + inline simdjson_result parse_many(const std::string &&s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) = delete;// unsafe /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ inline simdjson_result parse_many(const padded_string &s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; - inline simdjson_result parse_many(const padded_string &&s, size_t batch_size) = delete;// unsafe + inline simdjson_result parse_many(const padded_string &&s, size_t batch_size = dom::DEFAULT_BATCH_SIZE) = delete;// unsafe + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) + * + * Because padded_string_view guarantees SIMDJSON_PADDING trailing bytes, this + * overload is safe to use with buffers that the caller owns elsewhere (for + * example, a padded_memory_map), with no extra copy. Without this overload, + * passing a padded_string_view would silently bind to the padded_string + * overload via an implicit conversion, allocating and copying the input, and + * -- because that temporary is destroyed at the end of the full-expression -- + * leaving the returned document_stream pointing at freed memory. */ + inline simdjson_result parse_many(const padded_string_view &v, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept; /** @private We do not want to allow implicit conversion from C string to std::string. */ simdjson_result parse_many(const char *buf, size_t batch_size = dom::DEFAULT_BATCH_SIZE) noexcept = delete; + /** + * Parse a stream of JSON documents with explicit format specification. + * + * @param buf The concatenated JSON documents. + * @param len The length of the buffer. + * @param batch_size The batch size to use. + * @param format The stream format. + * @return A stream of documents, or an error. + */ + inline simdjson_result parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const std::string &s, size_t batch_size, stream_format format) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const padded_string_view &v, size_t batch_size, stream_format format) noexcept; + /** @private An rvalue input is destroyed at the end of the full-expression, while + * the returned document_stream only holds a pointer to it: iterating the stream would + * then read freed memory. These deleted overloads also catch a std::string_view + * argument, which would otherwise convert implicitly to a padded_string temporary. */ + inline simdjson_result parse_many(const std::string &&s, size_t batch_size, stream_format format) = delete;// unsafe + /** @private @overload parse_many(const std::string &&s, size_t batch_size, stream_format format) */ + inline simdjson_result parse_many(const padded_string &&s, size_t batch_size, stream_format format) = delete;// unsafe + /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length * and `max_depth` depth. @@ -6354,8 +7141,9 @@ class document_stream { * * IMPORTANT: this value is only meaningful under the conditions below. It is * computed from stage-1 bookkeeping, and outside these conditions it is not - * merely imprecise, it is arbitrary -- it can exceed size_in_bytes() or wrap - * around to a huge value. Check it only when both of the following hold: + * merely imprecise, it is + * arbitrary -- it can exceed size_in_bytes() or wrap around to a huge value. + * Check it only when all of the following hold: * * - you iterated all the way to the end of the stream; * - no document reported an error. Iteration stops at the first failed @@ -6364,6 +7152,9 @@ class document_stream { * If you need to know about a truncated tail outside those conditions, track * it yourself from the last successful document (see iterator::current_index() * and iterator::source()). + * + * An empty input (zero bytes) or an input made only of white space contains + * no document: truncated_bytes() returns zero. */ inline size_t truncated_bytes() const noexcept; /** @@ -6463,12 +7254,14 @@ class document_stream { * @param buf is the raw byte buffer we need to process * @param len is the length of the raw byte buffer in bytes * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document) + * @param format is the stream format */ simdjson_inline document_stream( dom::parser &parser, const uint8_t *buf, size_t len, - size_t batch_size + size_t batch_size, + stream_format format = stream_format::whitespace_delimited ) noexcept; /** @@ -6518,6 +7311,8 @@ class document_stream { const uint8_t *buf; size_t len; size_t batch_size; + /** The stream format. */ + stream_format format; /** The error (or lack thereof) from the current document. */ error_code error; size_t batch_start{0}; @@ -6605,6 +7400,8 @@ enum class element_type { STRING = '"', ///< std::string_view BOOL = 't', ///< bool NULL_VALUE = 'n', ///< null + /// The BIGINT type is for integers that do not fit in 64 bits. It is only present + // if you set parser.number_as_string(true). BIGINT = 'Z' ///< std::string_view: big integer stored as raw digit string }; @@ -6673,6 +7470,20 @@ class element { * Returns INCORRECT_TYPE if the JSON element is not a string. */ inline simdjson_result get_string() const noexcept; + + #if SIMDJSON_SUPPORTS_CHAR8_T + /** + * Cast this element to a C++20 UTF-8 string. + * + * The string is guaranteed to be valid UTF-8. + * + * @returns A std::u8string_view. The string is stored in the parser and will be invalidated the next time it + * parses a document or when it is destroyed. + * Returns INCORRECT_TYPE if the JSON element is not a string. + */ + inline simdjson_result get_u8string() const noexcept; + #endif + /** * Cast this element to a signed integer. * @@ -6778,7 +7589,7 @@ class element { * Supported types: * - Boolean: bool * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * + * - String: std::string_view, const char *, std::u8string_view (C++20) * - Array: dom::array * - Object: dom::object * @@ -6793,7 +7604,7 @@ class element { * Supported types: * - Boolean: bool * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * + * - String: std::string_view, const char *, std::u8string_view (C++20) * - Array: dom::array * - Object: dom::object * @@ -6823,7 +7634,7 @@ class element { * Supported types: * - Boolean: bool * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * + * - String: std::string_view, const char *, std::u8string_view (C++20) * - Array: dom::array * - Object: dom::object * @@ -6842,7 +7653,7 @@ class element { * Supported types: * - Boolean: bool * - Number: double, uint64_t, int64_t - * - String: std::string_view, const char * + * - String: std::string_view, const char *, std::u8string_view (C++20) * - Array: dom::array * - Object: dom::object * @@ -7126,6 +7937,9 @@ struct simdjson_result : public internal::simdjson_result_base get_c_str() const noexcept; simdjson_inline simdjson_result get_string_length() const noexcept; simdjson_inline simdjson_result get_string() const noexcept; + #if SIMDJSON_SUPPORTS_CHAR8_T + simdjson_inline simdjson_result get_u8string() const noexcept; + #endif simdjson_inline simdjson_result get_int64() const noexcept; simdjson_inline simdjson_result get_uint64() const noexcept; simdjson_inline simdjson_result get_double() const noexcept; @@ -7825,6 +8639,24 @@ template std::string prettify(simdjson_result x) { namespace simdjson { +/** Specifies where commas should be in table-formatted elements. */ +enum class table_comma_placement { + /** Commas come right after the value */ + before_padding, + /** Commas come after the column padding, so they line up in their own column. */ + after_padding, + /** Commas come right after the value, except for columns of numbers */ + before_padding_except_numbers, +}; + +/** Options for how lists or columns of numbers should be aligned */ +enum class number_list_alignment { + /** Left-aligns numbers */ + left, + /** Right-aligns numbers */ + right, +}; + /** * Configuration options for FracturedJson formatting. * @@ -7839,12 +8671,6 @@ struct fractured_json_options { */ size_t max_total_line_length = 120; - /** - * Maximum length for inlined elements (default: 80). - * Simple arrays/objects shorter than this may be rendered inline. - */ - size_t max_inline_length = 80; - /** * Maximum nesting depth for inline rendering (default: 2). * Elements with complexity exceeding this will be expanded. @@ -7853,11 +8679,11 @@ struct fractured_json_options { size_t max_inline_complexity = 2; /** - * Maximum complexity for compact array formatting (default: 1). + * Maximum complexity for compact array formatting (default: 2). * Arrays with elements of this complexity or less may have multiple * items per line. */ - size_t max_compact_array_complexity = 1; + size_t max_compact_array_complexity = 2; /** * Number of spaces per indentation level (default: 4). @@ -7865,23 +8691,26 @@ struct fractured_json_options { size_t indent_spaces = 4; /** - * Enable tabular formatting for arrays of similar objects (default: true). - * When enabled, arrays of objects with identical keys are formatted - * as aligned tables. + * Forces elements close to the root to always fully expand, regardless of other settings. + * (default: -1). -1 = none; 0 = root node only; 1 = root node and its children; etc. */ - bool enable_table_format = true; + int always_expand_depth = -1; /** - * Minimum number of rows to trigger table mode (default: 3). + * Enable tabular formatting for arrays of similar objects or arrays + * (default: true). When enabled, the rows of such an array are written one + * per line with their columns aligned. Rows need not have identical keys: + * columns are ordered by the first occurrence of each key, and a row + * missing a key gets blank space in that column. */ - size_t min_table_rows = 3; + bool enable_table_format = true; /** - * Similarity threshold for table detection (default: 0.8). - * Objects must share at least this fraction of keys to be formatted - * as a table. + * Maximum complexity of each row of a table (default: 2). + * 0 = rows may only be scalars (a single column); 1 = rows may be flat + * arrays/objects; higher values allow deeper nesting. */ - double table_similarity_threshold = 0.8; + size_t max_table_row_complexity = 2; /** * Enable compact multiline arrays (default: true). @@ -7891,16 +8720,26 @@ struct fractured_json_options { bool enable_compact_multiline = true; /** - * Maximum array items per line in compact mode (default: 10). + * Minimum number of items per line for an array to be formatted as a + * compact multiline array (default: 3). */ - size_t max_items_per_line = 10; + size_t min_compact_array_row_items = 3; /** - * Add space inside brackets for simple containers (default: true). - * When true: { "key": "value" } - * When false: {"key": "value"} + * Add space inside brackets for containers that hold only scalar values + * (default: false). When true: { "key": "value" }. When false: + * {"key": "value"}. + * @see nested_bracket_padding */ - bool simple_bracket_padding = true; + bool simple_bracket_padding = false; + + /** + * Add space inside brackets for containers that hold at least one + * nested array/object (default: true). When true: { "a": [1, 2] }. + * When false: {"a": [1, 2]}. + * @see simple_bracket_padding + */ + bool nested_bracket_padding = true; /** * Add space after colons (default: true). @@ -7915,6 +8754,18 @@ struct fractured_json_options { * When false: [1,2,3] */ bool comma_padding = true; + + /** + * Placement of commas relative to column padding in table-formatted rows + * and compact multiline arrays (default: before_padding_except_numbers). + */ + table_comma_placement comma_placement = table_comma_placement::before_padding_except_numbers; + + /** + * Controls alignment of numbers in table columns or compact multiline arrays + * (default: left). Numbers are always written exactly as in the input. + */ + number_list_alignment number_alignment = number_list_alignment::left; }; /** @@ -7995,12 +8846,55 @@ inline std::string fractured_json_string(std::string_view json_str, #ifndef SIMDJSON_JSONPATHUTIL_H #define SIMDJSON_JSONPATHUTIL_H +/* skipped duplicate #include "simdjson/error.h" */ #include /* skipped duplicate #include "simdjson/common_defs.h" */ +#include #include namespace simdjson { +namespace internal { +/** + * Parses the next JSON Pointer array index token. + * + * The caller passes a pointer fragment with no leading '/', such as "123/foo". + * On success, array_index receives the parsed index and token_length receives + * the number of bytes consumed before the next '/' or the end of the fragment. + */ +simdjson_inline error_code parse_json_pointer_array_index(std::string_view json_pointer, + size_t &array_index, + size_t &token_length) noexcept { + array_index = 0; + token_length = 0; + + for (; token_length < json_pointer.length() && json_pointer[token_length] != '/'; + token_length++) { + uint8_t digit = uint8_t(json_pointer[token_length] - '0'); + // Check for non-digit in array index. If it's there, we're trying to get a field in an object. + if (digit > 9) { + return INCORRECT_TYPE; + } + // 0 followed by other digits is invalid. + if (token_length > 0 && json_pointer[0] == '0') { + return INVALID_JSON_POINTER; + } + if (array_index > + (((std::numeric_limits::max)() - digit) / 10)) { + return INDEX_OUT_OF_BOUNDS; + } + array_index = array_index * 10 + digit; + } + + // Empty string is invalid; so is a "/" with no digits before it. + if (token_length == 0) { + return INVALID_JSON_POINTER; + } + + return SUCCESS; +} +} // namespace internal + /** * Converts JSONPath to JSON Pointer. * @param json_path The JSONPath string to be converted. @@ -8220,6 +9114,72 @@ inline size_t tape_ref::after_element() const noexcept { simdjson_inline tape_type tape_ref::tape_ref_type() const noexcept { return static_cast(doc->tape[json_index] >> 56); } +simdjson_inline size_t tape_ref::before_element(size_t array_start) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(usable()); + SIMDJSON_DEVELOPMENT_ASSERT(json_index > array_start); + tape_ref previous(doc, json_index - 1); + if (previous.json_index == array_start) { return array_start; } + // An exact numeric marker cannot end an element unless it is itself the + // payload of a number. In that case its header is immediately before it. + if (previous.is_int64() || previous.is_uint64() || previous.is_double()) { + return previous.json_index - 1; + } + tape_ref probe(doc, previous.json_index - 1); + + // Validate both container links before examining its contents. A candidate + // opening tag preceded by an even run of numeric markers is a real tag, + // not a numeric payload. Scan both candidate boundaries together so that a + // forged opening tag inside a nested value cannot cause an unbounded detour. + // For a real container, the opening probe visits preceding siblings. For a + // numeric payload, the other probe does. Stopping at the shorter run bounds + // the work by the array's immediate elements, rather than nested contents. + const auto type = previous.tape_ref_type(); + if (type == tape_type::END_ARRAY || type == tape_type::END_OBJECT) { + const size_t start = previous.matching_brace_index(); + if (start > array_start && start < previous.json_index) { + tape_ref opening(doc, start); + const auto expected = type == tape_type::END_ARRAY + ? tape_type::START_ARRAY : tape_type::START_OBJECT; + if (opening.tape_ref_type() == expected && + opening.matching_brace_index() == previous.json_index + 1) { + tape_ref before_opening(doc, start - 1); + while ((before_opening.is_int64() || before_opening.is_uint64() || + before_opening.is_double()) && + (probe.is_int64() || probe.is_uint64() || probe.is_double())) { + --before_opening.json_index; + --probe.json_index; + } + if (!before_opening.is_int64() && !before_opening.is_uint64() && + !before_opening.is_double() && + (start - before_opening.json_index) % 2 == 1) { + return start; + } + } + } + } + + // Numeric payloads can have ANY bit pattern, including another type's tag. + // A run of exact numeric markers starts with a header, then alternates + // between payload and header. An odd run before this word makes it a payload. + // Subsequent reverse increments through exact-marker payloads take the + // constant-time numeric-payload branch above. + while (probe.is_int64() || probe.is_uint64() || probe.is_double()) { + --probe.json_index; + } + if ((previous.json_index - probe.json_index) % 2 == 0) { + return previous.json_index - 1; + } + + // Once distinguished from numeric payloads, closing container tags link + // directly back to their opening tags. + switch (previous.tape_ref_type()) { + case tape_type::END_ARRAY: + case tape_type::END_OBJECT: + return previous.matching_brace_index(); + default: + return previous.json_index; + } +} simdjson_inline uint64_t internal::tape_ref::tape_value() const noexcept { return doc->tape[json_index] & internal::JSON_VALUE_MASK; } @@ -8295,6 +9255,14 @@ inline size_t simdjson_result::size() const noexcept(false) { if (error()) { throw simdjson_error(error()); } return first.size(); } +inline dom::array::reverse_iterator simdjson_result::rbegin() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.rbegin(); +} +inline dom::array::reverse_iterator simdjson_result::rend() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.rend(); +} #endif // SIMDJSON_EXCEPTIONS @@ -8304,6 +9272,7 @@ inline simdjson_result simdjson_result::at_pointer(std } inline simdjson_result simdjson_result::at_path(std::string_view json_path) const noexcept { + if (error()) { return error(); } auto json_pointer = json_path_to_pointer_conversion(json_path); if (json_pointer == "-1") { return INVALID_JSON_POINTER; } return at_pointer(json_pointer); @@ -8344,6 +9313,15 @@ inline size_t array::size() const noexcept { SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return tape.scope_count(); } +inline array::reverse_iterator array::rbegin() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); + const internal::tape_ref end_tape(tape.doc, tape.matching_brace_index() - 1); + return reverse_iterator(internal::tape_ref(tape.doc, end_tape.before_element(tape.json_index)), tape.json_index); +} +inline array::reverse_iterator array::rend() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); + return reverse_iterator(tape, tape.json_index); +} inline size_t array::number_of_slots() const noexcept { SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 return tape.matching_brace_index() - tape.json_index; @@ -8360,21 +9338,9 @@ inline simdjson_result array::at_pointer(std::string_view json_pointer) // We don't support this, because we're returning a real element, not a position. if (json_pointer == "-") { return INDEX_OUT_OF_BOUNDS; } - // Read the array index size_t array_index = 0; size_t i; - for (i = 0; i < json_pointer.length() && json_pointer[i] != '/'; i++) { - uint8_t digit = uint8_t(json_pointer[i] - '0'); - // Check for non-digit in array index. If it's there, we're trying to get a field in an object - if (digit > 9) { return INCORRECT_TYPE; } - array_index = array_index*10 + digit; - } - - // 0 followed by other digits is invalid - if (i > 1 && json_pointer[0] == '0') { return INVALID_JSON_POINTER; } // "JSON pointer array index has other characters after 0" - - // Empty string is invalid; so is a "/" with no digits before it - if (i == 0) { return INVALID_JSON_POINTER; } // "Empty string in JSON pointer array index" + SIMDJSON_TRY(internal::parse_json_pointer_array_index(json_pointer, array_index, i)); // Get the child auto child = array(tape).at(array_index); @@ -8409,7 +9375,6 @@ inline void array::process_json_path_of_child_elements(std::vector::ite if(error) { continue; } - accumulator.reserve(accumulator.size() + child_result.size()); accumulator.insert(accumulator.end(), std::make_move_iterator(child_result.begin()), std::make_move_iterator(child_result.end())); @@ -8505,6 +9470,30 @@ inline array::operator element() const noexcept { return element(tape); } +// +// array::reverse_iterator inline implementation +// +simdjson_inline array::reverse_iterator::reverse_iterator(const internal::tape_ref &_tape, size_t _array_start) noexcept + : tape{_tape}, array_start{_array_start} { } +inline element array::reverse_iterator::operator*() const noexcept { + return element(tape); +} +inline array::reverse_iterator& array::reverse_iterator::operator++() noexcept { + tape.json_index = tape.before_element(array_start); + return *this; +} +inline array::reverse_iterator array::reverse_iterator::operator++(int) noexcept { + reverse_iterator out = *this; + ++*this; + return out; +} +inline bool array::reverse_iterator::operator==(const reverse_iterator& other) const noexcept { + return tape.doc == other.tape.doc && tape.json_index == other.tape.json_index; +} +inline bool array::reverse_iterator::operator!=(const reverse_iterator& other) const noexcept { + return !(*this == other); +} + // // array::iterator inline implementation // @@ -8596,6 +9585,7 @@ inline simdjson_result simdjson_result::at_pointer(st return first.at_pointer(json_pointer); } inline simdjson_result simdjson_result::at_path(std::string_view json_path) const noexcept { + if (error()) { return error(); } auto json_pointer = json_path_to_pointer_conversion(json_path); if (json_pointer == "-1") { return INVALID_JSON_POINTER; } return at_pointer(json_pointer); @@ -8725,7 +9715,6 @@ inline void object::process_json_path_of_child_elements(std::vector::it if(error) { continue; } - accumulator.reserve(accumulator.size() + child_result.size()); accumulator.insert(accumulator.end(), std::make_move_iterator(child_result.begin()), std::make_move_iterator(child_result.end())); @@ -9003,6 +9992,12 @@ simdjson_inline simdjson_result simdjson_result: if (error()) { return error(); } return first.get_string(); } +#if SIMDJSON_SUPPORTS_CHAR8_T +simdjson_inline simdjson_result simdjson_result::get_u8string() const noexcept { + if (error()) { return error(); } + return first.get_u8string(); +} +#endif simdjson_inline simdjson_result simdjson_result::get_int64() const noexcept { if (error()) { return error(); } return first.get_int64(); @@ -9069,6 +10064,7 @@ simdjson_inline simdjson_result simdjson_result::at_ return first.at_pointer(json_pointer); } simdjson_inline simdjson_result simdjson_result::at_path(const std::string_view json_path) const noexcept { + if (error()) { return error(); } auto json_pointer = json_path_to_pointer_conversion(json_path); if (json_pointer == "-1") { return INVALID_JSON_POINTER; } return at_pointer(json_pointer); @@ -9201,6 +10197,13 @@ inline simdjson_result element::get_string() const noexcept { return INCORRECT_TYPE; } } +#if SIMDJSON_SUPPORTS_CHAR8_T +inline simdjson_result element::get_u8string() const noexcept { + std::string_view v; + SIMDJSON_TRY(get_string().get(v)); + return internal::as_u8string_view(v); +} +#endif inline simdjson_result element::get_uint64() const noexcept { SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 if(simdjson_unlikely(!tape.is_uint64())) { // branch rarely taken @@ -9296,6 +10299,9 @@ template<> inline simdjson_result element::get() const noexcept { template<> inline simdjson_result element::get() const noexcept { return get_object(); } template<> inline simdjson_result element::get() const noexcept { return get_c_str(); } template<> inline simdjson_result element::get() const noexcept { return get_string(); } +#if SIMDJSON_SUPPORTS_CHAR8_T +template<> inline simdjson_result element::get() const noexcept { return get_u8string(); } +#endif template<> inline simdjson_result element::get() const noexcept { return get_int64(); } template<> inline simdjson_result element::get() const noexcept { return get_uint64(); } template<> inline simdjson_result element::get() const noexcept { return get_double(); } @@ -9628,6 +10634,29 @@ inline simdjson_result parser::parse_into_document(document& provided_d return provided_doc.root(); } +inline simdjson_result parser::parse_into_document_unpadded(document& provided_doc, const uint8_t *buf, size_t len) & noexcept { + // Like parse_into_document with realloc_if_needed=false (no copy, parse in + // place), but we tell the implementation the buffer is not padded so stage 2 + // avoids reading past buf+len: string unescaping is bounded, near-the-end + // numbers are parsed from a padded copy, and atoms use length-aware + // validators (see tape_builder). Stage 1 is already safe for unpadded input. + error_code _error = ensure_capacity(provided_doc, len); + if (_error) { return _error; } + + if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { + buf += 3; + len -= 3; + } + implementation->_number_as_string = _number_as_string; + implementation->_unpadded = true; + _error = implementation->parse(buf, len, provided_doc); + implementation->_unpadded = false; // restore so later padded parses use the fast path + + if (_error) { return _error; } + + return provided_doc.root(); +} + simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const char *buf, size_t len, bool realloc_if_needed) & noexcept { return parse_into_document(provided_doc, reinterpret_cast(buf), len, realloc_if_needed); } @@ -9656,13 +10685,18 @@ simdjson_inline simdjson_result parser::parse(const padded_string_view return parse(v.data(), v.length(), false); } +inline simdjson_result parser::parse_unpadded(const uint8_t *buf, size_t len) & noexcept { + return parse_into_document_unpadded(doc, buf, len); +} +simdjson_inline simdjson_result parser::parse_unpadded(const char *buf, size_t len) & noexcept { + return parse_unpadded(reinterpret_cast(buf), len); +} +simdjson_inline simdjson_result parser::parse_unpadded(std::string_view s) & noexcept { + return parse_unpadded(reinterpret_cast(s.data()), s.size()); +} + inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size) noexcept { - if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } - if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { - buf += 3; - len -= 3; - } - return document_stream(*this, buf, len, batch_size); + return parse_many(buf, len, batch_size, stream_format::whitespace_delimited); } inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size) noexcept { return parse_many(reinterpret_cast(buf), len, batch_size); @@ -9673,6 +10707,48 @@ inline simdjson_result parser::parse_many(const std::string &s, inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size) noexcept { return parse_many(s.data(), s.length(), batch_size); } +inline simdjson_result parser::parse_many(const padded_string_view &v, size_t batch_size) noexcept { + return parse_many(v.data(), v.length(), batch_size); +} + +inline simdjson_result parser::parse_many(const uint8_t *buf, size_t len, size_t batch_size, stream_format format) noexcept { + if(batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { + buf += 3; + len -= 3; + } + if (format == stream_format::comma_delimited_array) { + // Strip leading JSON whitespace. + while (len > 0 && (buf[0] == ' ' || buf[0] == '\t' || buf[0] == '\n' || buf[0] == '\r')) { + buf++; len--; + } + // Expect the opening '['. + if (len == 0 || buf[0] != '[') { return TAPE_ERROR; } + buf++; len--; + // Strip trailing JSON whitespace. + while (len > 0 && (buf[len-1] == ' ' || buf[len-1] == '\t' || buf[len-1] == '\n' || buf[len-1] == '\r')) { + len--; + } + // Expect the closing ']'. + if (len == 0 || buf[len-1] != ']') { return TAPE_ERROR; } + len--; + // Fall through to comma_delimited over the array contents. + format = stream_format::comma_delimited; + } + return document_stream(*this, buf, len, batch_size, format); +} +inline simdjson_result parser::parse_many(const char *buf, size_t len, size_t batch_size, stream_format format) noexcept { + return parse_many(reinterpret_cast(buf), len, batch_size, format); +} +inline simdjson_result parser::parse_many(const std::string &s, size_t batch_size, stream_format format) noexcept { + return parse_many(s.data(), s.length(), batch_size, format); +} +inline simdjson_result parser::parse_many(const padded_string &s, size_t batch_size, stream_format format) noexcept { + return parse_many(s.data(), s.length(), batch_size, format); +} +inline simdjson_result parser::parse_many(const padded_string_view &v, size_t batch_size, stream_format format) noexcept { + return parse_many(v.data(), v.length(), batch_size, format); +} simdjson_inline size_t parser::capacity() const noexcept { return implementation ? implementation->capacity() : 0; @@ -9830,12 +10906,14 @@ simdjson_inline document_stream::document_stream( dom::parser &_parser, const uint8_t *_buf, size_t _len, - size_t _batch_size + size_t _batch_size, + stream_format _format ) noexcept : parser{&_parser}, buf{_buf}, len{_len}, batch_size{_batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size}, + format{_format}, error{SUCCESS} #ifdef SIMDJSON_THREADS_ENABLED , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change @@ -9853,6 +10931,7 @@ simdjson_inline document_stream::document_stream() noexcept buf{nullptr}, len{0}, batch_size{0}, + format{stream_format::whitespace_delimited}, error{UNINITIALIZED} #ifdef SIMDJSON_THREADS_ENABLED , use_thread(false) @@ -9929,6 +11008,7 @@ inline void document_stream::start() noexcept { if (error) { return; } error = parser->ensure_capacity(batch_size); if (error) { return; } + parser->implementation->_number_as_string = parser->number_as_string(); // Always run the first stage 1 parse immediately batch_start = 0; error = run_stage1(*parser, batch_start); @@ -9968,7 +11048,40 @@ simdjson_inline std::string_view document_stream::iterator::source() const noexc } else { size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index]; size_t svlen = next_doc_index - current_index(); - while(svlen > 1 && (std::isspace(start[svlen-1]) || start[svlen-1] == '\0')) { + // When the scalar is followed by a truncated document, the structural + // indexes of that document were dropped and next_doc_index is the end of + // the input, so we bound the scalar by scanning the token itself. + size_t token_len = 0; + if (*start == '"') { + token_len = 1; + while (token_len < svlen) { + char c = start[token_len++]; + if (c == '\\') { + token_len++; + } else if (c == '"') { + break; + } + } + } else { + while (token_len < svlen) { + char c = start[token_len]; + if (std::isspace(static_cast(c)) || c == ',' || c == '{' || c == '[' || c == '\0' || static_cast(c) == 0x1E) { + break; + } + token_len++; + } + } + if (token_len > 0 && token_len < svlen) { + svlen = token_len; + } + // Trim trailing whitespace, NUL, and RS (0x1E). In RFC 7464 json_sequence + // mode the scanner classifies RS as a scalar character, so an RS-prefixed + // scalar document (number/true/false/null/string) has no closing structural + // index and the slice runs all the way up to the next document's RS. RS + // cannot legally appear in a JSON value at the source level (control + // characters in strings must be escaped as \u001E), so stripping it is + // safe in every stream_format. + while(svlen > 1 && (std::isspace(static_cast(start[svlen-1])) || start[svlen-1] == '\0' || static_cast(start[svlen-1]) == 0x1E || (stream->format == stream_format::comma_delimited && start[svlen-1] == ','))) { svlen--; } return std::string_view(start, svlen); @@ -10008,6 +11121,9 @@ inline size_t document_stream::size_in_bytes() const noexcept { } inline size_t document_stream::truncated_bytes() const noexcept { + // Stage 1 returns EMPTY on zero-length input before it writes the index + // sentinels read below, so they would still hold a previous stream's values. + if (len == 0) { return 0; } if(error == CAPACITY) { return len - batch_start; } return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1]; } @@ -10018,10 +11134,35 @@ inline size_t document_stream::next_batch_start() const noexcept { inline error_code document_stream::run_stage1(dom::parser &p, size_t _batch_start) noexcept { size_t remaining = len - _batch_start; + stage1_mode mode; if (remaining <= batch_size) { - return p.implementation->stage1(&buf[_batch_start], remaining, stage1_mode::streaming_final); + // Final batch + switch (format) { + case stream_format::json_sequence: + mode = stage1_mode::json_sequence_final; + break; + case stream_format::comma_delimited: + mode = stage1_mode::comma_delimited_final; + break; + default: + mode = stage1_mode::streaming_final; + break; + } + return p.implementation->stage1(&buf[_batch_start], remaining, mode); } else { - return p.implementation->stage1(&buf[_batch_start], batch_size, stage1_mode::streaming_partial); + // Partial batch + switch (format) { + case stream_format::json_sequence: + mode = stage1_mode::json_sequence_partial; + break; + case stream_format::comma_delimited: + mode = stage1_mode::comma_delimited_partial; + break; + default: + mode = stage1_mode::streaming_partial; + break; + } + return p.implementation->stage1(&buf[_batch_start], batch_size, mode); } } @@ -10194,16 +11335,25 @@ inline error_code document::allocate(size_t capacity) noexcept { allocated_capacity = 0; return SUCCESS; } + if (capacity > SIMDJSON_MAXSIZE_BYTES) { + return CAPACITY; + } // a pathological input like "[[[[..." would generate capacity tape elements, so // need a capacity of at least capacity + 1, but it is also possible to do // worse with "[7,7,7,7,6,7,7,7,6,7,7,6,[7,7,7,7,6,7,7,7,6,7,7,6,7,7,7,7,7,7,6" //where capacity + 1 tape elements are // generated, see issue https://github.com/simdjson/simdjson/issues/345 + if(capacity + 3 < capacity) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } size_t tape_capacity = SIMDJSON_ROUNDUP_N(capacity + 3, 64); // a document with only zero-length strings... could have capacity/3 string // and we would need capacity/3 * 5 bytes on the string buffer - size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * capacity / 3 + SIMDJSON_PADDING, 64); + if(5 * (capacity / 3) + SIMDJSON_PADDING < SIMDJSON_PADDING) { + return CAPACITY; // overflow, only happen on legacy 32-bit systems with very large capacity + } + size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * (capacity / 3) + SIMDJSON_PADDING, 64); string_buf.reset( new (std::nothrow) uint8_t[string_capacity]); tape.reset(new (std::nothrow) uint64_t[tape_capacity]); if(!(string_buf && tape)) { @@ -10347,6 +11497,7 @@ inline bool document::dump_raw_tape(std::ostream &os) const noexcept { /* skipped duplicate #include "simdjson/dom/object-inl.h" */ /* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ +#include #include namespace simdjson { @@ -10517,11 +11668,29 @@ simdjson_inline void base_formatter::number(int64_t x) { template simdjson_inline void base_formatter::number(double x) { - char number_buffer[24]; +#if SIMDJSON_ENABLE_NAN_INF + if (simdjson_unlikely(!std::isfinite(x))) { + if (std::isnan(x)) { + char const *s = "NaN"; + chars(s, s + 3); + } else { + if (x < 0) { + one_char('-'); + } + char const *s = "Infinity"; + chars(s, s + 8); + } + return; + } +#endif + + // Must be to_chars_buffer_size (40): only ~24 chars are emitted, but + // to_chars over-writes with fixed-size 16/17-byte copies for inlining. + char number_buffer[simdjson::internal::to_chars_buffer_size]; // Currently, passing the nullptr to the second argument is // safe because our implementation does not check the second // argument. - char *newp = internal::to_chars(number_buffer, nullptr, x); + char *newp = simdjson::internal::to_chars(number_buffer, nullptr, x); chars(number_buffer, newp); } @@ -10797,7 +11966,7 @@ inline void string_builder::append(simdjson::dom::element value) { format.string(iter.get_string_view()); break; case tape_type::BIGINT: { - // Big integer stored as string — output raw digits (no quotes) + // Big integer stored as string -- output raw digits (no quotes) auto sv = iter.get_string_view(); format.chars(sv.data(), sv.data() + sv.size()); break; @@ -10935,7 +12104,7 @@ simdjson_inline std::string_view string_builder::str() const { #include #include #include -#include +#include namespace simdjson { namespace internal { @@ -10944,12 +12113,48 @@ namespace internal { * Layout mode for fractured JSON formatting. */ enum class layout_mode { - INLINE, // Single line: [1, 2, 3] or {"a": 1} - COMPACT_MULTILINE, // Multiple items per line with breaks - TABLE, // Tabular format for arrays of similar objects - EXPANDED // Traditional multi-line with indentation + single_line, // Single line: [1, 2, 3] or {"a": 1} + compact_multiline, // Multiple items per line with breaks + table, // Tabular format for arrays of similar objects + expanded // Traditional multi-line with indentation +}; + +/** Kind of value found in a table column across all rows that have one. */ +enum class table_column_type { + unknown, + simple, // string, bool, or null + number, + array, + object, + mixed // rows disagree on kind +}; + +/** Column of a table-formatted array.*/ +struct table_column { + /** Column name for object rows; empty for array rows. */ + std::string key{}; + /** Rendered length of key */ + size_t key_width = 0; + table_column_type type = table_column_type::unknown; + /** Widest rendered value in this column (when fully expanding all children) */ + size_t width = 0; + /** Widest plain value in this column. */ + size_t plain_width = 0; + /** subcolumns (only populated if every child is an array or every child is an object) */ + std::vector children{}; }; +/** Whether rows from this column list use nested vs. simple bracket padding + * One shared decision, since picking it per-row would misalign width-aligned rows. */ +inline bool table_row_is_nested(const std::vector& columns) { + for (const table_column& col : columns) { + if (col.type == table_column_type::object || col.type == table_column_type::array) { + return true; + } + } + return false; +} + /** * Metrics computed for a JSON element during structure analysis. * These metrics drive layout decisions and contain child metrics for recursive formatting. @@ -10970,16 +12175,32 @@ struct element_metrics { /** Is this an array where all elements have similar structure? */ bool is_uniform_array = false; - /** For uniform arrays of objects: the common keys */ - std::vector common_keys{}; - - /** Recommended layout mode based on analysis */ - layout_mode recommended_layout = layout_mode::EXPANDED; + /** for uniform arrays: this array's columns for alignment */ + std::vector table_columns{}; + /** Widest table row after pruning recursive columns that don't fit into line budget */ + size_t table_row_width = 0; + /** Widest table row without pruning recursive columns that don't fit into line budget */ + size_t table_row_width_full = 0; /** Child metrics for arrays and objects (in order of iteration) */ std::vector children{}; + + /** For scalar uniform arrays (table_columns empty): the rows' common type. */ + table_column_type scalar_column_type = table_column_type::unknown; }; +/** children[idx], or a default-constructed element_metrics if idx is out of range */ +inline const element_metrics& child_metrics_at(const std::vector& children, size_t idx) { + static const element_metrics empty{}; + return idx < children.size() ? children[idx] : empty; +} + +/** *ptr, or a default-constructed element_metrics if ptr is null. */ +inline const element_metrics& child_metrics_at(const element_metrics* ptr) { + static const element_metrics empty{}; + return ptr ? *ptr : empty; +} + /** * Analyzes JSON structure to compute metrics for formatting decisions. * @@ -11040,20 +12261,27 @@ class structure_analyzer { element_metrics analyze_object(const dom::object& obj, const fractured_json_options& opts); + /** Decide layout at the given render depth. Kept out of analysis since + * the same metrics can render inline or expanded at different depths. */ + static layout_mode decide_layout(const element_metrics& metrics, + size_t depth, + const fractured_json_options& opts, + bool has_trailing_comma = false); + private: const fractured_json_options* current_opts_ = nullptr; /** Recursive analysis implementation */ - element_metrics analyze_element(const dom::element& elem, size_t depth); + element_metrics analyze_element(const dom::element& elem, size_t depth) const; /** Analyze scalar values (strings, numbers, booleans, null) */ - element_metrics analyze_scalar(const dom::element& elem); + element_metrics analyze_scalar(const dom::element& elem) const; /** Analyze an array element */ - element_metrics analyze_array(const dom::array& arr, size_t depth); + element_metrics analyze_array(const dom::array& arr, size_t depth) const; /** Analyze an object element */ - element_metrics analyze_object(const dom::object& obj, size_t depth); + element_metrics analyze_object(const dom::object& obj, size_t depth) const; /** Estimate inline length for a string (including quotes and escaping) */ size_t estimate_string_length(std::string_view s) const; @@ -11064,27 +12292,35 @@ class structure_analyzer { size_t estimate_number_length(uint64_t u) const; /** - * Check if an array contains uniform objects suitable for table formatting. + * Check if an array contains uniformly-shaped rows suitable for table + * formatting, filling in corresponding metrics * @param arr The array to check - * @param common_keys Output: keys common to all objects - * @return true if the array is suitable for table formatting + * @param metrics The array's metrics, with children already filled in + * @param depth The array's depth */ - bool check_array_uniformity(const dom::array& arr, - std::vector& common_keys) const; + bool check_array_uniformity(const dom::array& arr, element_metrics& metrics, size_t depth) const; - /** - * Compute similarity between two objects. - * @return Fraction of keys that are common (0.0 to 1.0) - */ - double compute_object_similarity(const dom::object& a, - const dom::object& b) const; + static table_column_type classify_table_value(dom::element_type type); - /** - * Decide the recommended layout mode based on metrics and options. - */ - layout_mode decide_layout(const element_metrics& metrics, - size_t depth, - size_t available_width) const; + /** Find common type across a set of sibling values and the widest of their rendered lengths */ + static void classify_and_measure(const std::vector>& values, + table_column_type& common, size_t& max_width); + + /** Recursively build table columns for an array */ + void build_table_columns(const std::vector>& values, + std::vector& out_columns) const; + + /** Rendered width of a row, assuming each column's current */ + size_t compute_columns_width(const std::vector& columns) const; + + /** Height of a column's recursion (0 = leaf). */ + static size_t column_height(const table_column& column); + + /** Flatten deepest columns of the table */ + static bool flatten_deepest_columns(std::vector& columns); + + /** Bottom-up refresh of column widths after flatten_deepest_columns. */ + void recompute_column_widths(std::vector& columns) const; }; } // namespace internal @@ -11130,56 +12366,35 @@ class fractured_formatter : public base_formatter { /** Get the current layout mode */ layout_mode get_layout_mode() const; - /** Set current depth for formatting decisions */ - void set_depth(size_t depth); - - /** Get current depth */ - size_t get_depth() const; - /** Track current line length for compact multiline decisions */ void track_line_length(size_t chars); - /** Reset line length (after newline) */ - void reset_line_length(); - - /** Get current line length */ - size_t get_line_length() const; - /** Check if we should break to a new line in compact mode */ bool should_break_line(size_t upcoming_length) const; /** Get the options */ const fractured_json_options& options() const; - // Table formatting support - /** Begin a table row */ - void begin_table_row(); - - /** End a table row */ - void end_table_row(); - - /** Set column widths for table alignment */ - void set_column_widths(const std::vector& widths); - - /** Get current column index in table mode */ - size_t get_column_index() const; - - /** Advance to next column */ - void next_column(); - - /** Add padding to align with column width */ - void align_to_column_width(size_t actual_width); - private: fractured_json_options options_; - layout_mode current_layout_ = layout_mode::EXPANDED; - size_t current_depth_ = 0; + layout_mode current_layout_ = layout_mode::expanded; size_t current_line_length_ = 0; +}; + +/** RAII helper forcing single line layout and restoring previous mode on exit */ +class scoped_single_line_mode { +public: + explicit scoped_single_line_mode(fractured_formatter& format) + : format_(format), prev_(format.get_layout_mode()) { + format_.set_layout_mode(layout_mode::single_line); + } + ~scoped_single_line_mode() { format_.set_layout_mode(prev_); } + scoped_single_line_mode(const scoped_single_line_mode&) = delete; + scoped_single_line_mode& operator=(const scoped_single_line_mode&) = delete; - // Table state - bool in_table_mode_ = false; - std::vector column_widths_; - size_t current_column_ = 0; +private: + fractured_formatter& format_; + layout_mode prev_; }; /** @@ -11214,10 +12429,12 @@ class fractured_string_builder { fractured_json_options options_; /** Format an element using pre-computed metrics */ - void format_element(const dom::element& elem, const element_metrics& metrics, size_t depth); + void format_element(const dom::element& elem, const element_metrics& metrics, size_t depth, + bool has_trailing_comma = false); /** Format an array with the appropriate layout */ - void format_array(const dom::array& arr, const element_metrics& metrics, size_t depth); + void format_array(const dom::array& arr, const element_metrics& metrics, size_t depth, + bool has_trailing_comma = false); /** Format an array inline: [1, 2, 3] */ void format_array_inline(const dom::array& arr, const element_metrics& metrics); @@ -11225,14 +12442,51 @@ class fractured_string_builder { /** Format an array with compact multiline: multiple items per line */ void format_array_compact_multiline(const dom::array& arr, const element_metrics& metrics, size_t depth); + /** Like format_array_compact_multiline, but rows are cross-row aligned + * and packed using a fixed per-row slot width. */ + void format_array_compact_multiline_aligned(const dom::array& arr, const element_metrics& metrics, size_t depth); + /** Format an array as a table */ void format_array_as_table(const dom::array& arr, const element_metrics& metrics, size_t depth); + /** Write one object row's columns */ + void format_table_object_row(const dom::object& obj, const element_metrics& row_metrics, + const std::vector& columns, size_t depth); + + /** Write one array row's columns */ + void format_table_array_row(const dom::array& arr, const element_metrics& row_metrics, + const std::vector& columns, size_t depth); + + /** Dispatches to format_table_object_row/format_table_array_row based on elem's type. */ + void format_table_row(const dom::element& elem, const element_metrics& row_metrics, + const std::vector& columns, size_t depth); + + /** Row for a uniform scalar array: writes elem inline, then pads to width so every row lines up. */ + void format_table_scalar_row(const dom::element& elem, const element_metrics& row_metrics, + size_t width, size_t depth, table_column_type column_type); + + /** Shared per-column writer: recurses if the column has children, + * otherwise writes a plain padded value or blank. */ + void format_table_row_columns(const std::vector& columns, + const std::vector& found, + const std::vector& values, + const std::vector& value_metrics, + size_t depth); + + /** Writes a single aligned leaf value */ + void format_table_leaf_value(const dom::element& elem, const element_metrics& vm, size_t width, + table_column_type column_type, bool needs_comma, + bool add_comma_space, size_t depth); + + /** Whether, for a column of the given type, the comma goes right after the value */ + bool comma_goes_before_padding(table_column_type column_type) const; + /** Format an array expanded: one item per line */ void format_array_expanded(const dom::array& arr, const element_metrics& metrics, size_t depth); /** Format an object with the appropriate layout */ - void format_object(const dom::object& obj, const element_metrics& metrics, size_t depth); + void format_object(const dom::object& obj, const element_metrics& metrics, size_t depth, + bool has_trailing_comma = false); /** Format an object inline: {"a": 1, "b": 2} */ void format_object_inline(const dom::object& obj, const element_metrics& metrics); @@ -11243,12 +12497,8 @@ class fractured_string_builder { /** Format a scalar value */ void format_scalar(const dom::element& elem); - /** Calculate column widths for table formatting */ - std::vector calculate_column_widths(const dom::array& arr, - const std::vector& columns) const; - - /** Measure the actual formatted length of a value (for alignment) */ - size_t measure_value_length(const dom::element& elem) const; + /** Whether to pad this container's own brackets. */ + bool bracket_padding_for(const element_metrics& metrics) const; }; } // namespace internal @@ -11260,6 +12510,8 @@ class fractured_string_builder { #include #include #include +#include +#include namespace simdjson { namespace internal { @@ -11290,7 +12542,7 @@ inline element_metrics structure_analyzer::analyze_object(const dom::object& obj return analyze_object(obj, 0); } -inline element_metrics structure_analyzer::analyze_element(const dom::element& elem, size_t depth) { +inline element_metrics structure_analyzer::analyze_element(const dom::element& elem, size_t depth) const { switch (elem.type()) { case dom::element_type::ARRAY: { dom::array arr; @@ -11313,12 +12565,11 @@ inline element_metrics structure_analyzer::analyze_element(const dom::element& e return element_metrics{}; } -inline element_metrics structure_analyzer::analyze_scalar(const dom::element& elem) { +inline element_metrics structure_analyzer::analyze_scalar(const dom::element& elem) const { element_metrics metrics; metrics.complexity = 0; metrics.child_count = 0; metrics.can_inline = true; - metrics.recommended_layout = layout_mode::INLINE; switch (elem.type()) { case dom::element_type::STRING: { @@ -11367,7 +12618,7 @@ inline element_metrics structure_analyzer::analyze_scalar(const dom::element& el } inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, - size_t depth) { + size_t depth) const { element_metrics metrics; metrics.complexity = 1; // At least 1 for being an array metrics.estimated_inline_len = 2; // "[]" @@ -11392,35 +12643,31 @@ inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, // Complexity is 1 + max child complexity metrics.complexity = 1 + max_child_complexity; - // Check if can inline - metrics.can_inline = (metrics.complexity <= current_opts_->max_inline_complexity) && - (metrics.estimated_inline_len <= current_opts_->max_inline_length); - - // Check for uniform array (table formatting) - if (current_opts_->enable_table_format && - metrics.child_count >= current_opts_->min_table_rows) { - metrics.is_uniform_array = check_array_uniformity(arr, metrics.common_keys); + // Bracket padding "[ 1, 2 ]" vs "[1, 2]" + bool use_bracket_padding = (max_child_complexity >= 1) + ? current_opts_->nested_bracket_padding : current_opts_->simple_bracket_padding; + if (use_bracket_padding && metrics.child_count > 0) { + metrics.estimated_inline_len += 2; } - // Decide layout - if (metrics.child_count == 0) { - metrics.recommended_layout = layout_mode::INLINE; - } else if (metrics.can_inline) { - metrics.recommended_layout = layout_mode::INLINE; - } else if (metrics.is_uniform_array && !metrics.common_keys.empty()) { - metrics.recommended_layout = layout_mode::TABLE; - } else if (current_opts_->enable_compact_multiline && - max_child_complexity <= current_opts_->max_compact_array_complexity) { - metrics.recommended_layout = layout_mode::COMPACT_MULTILINE; - } else { - metrics.recommended_layout = layout_mode::EXPANDED; + // Check if can inline + metrics.can_inline = metrics.complexity <= current_opts_->max_inline_complexity; + + // Check for uniform array (table formatting, or aligned compact multiline). + bool wants_table_columns = + (current_opts_->enable_table_format && + max_child_complexity <= current_opts_->max_table_row_complexity) || + (current_opts_->enable_compact_multiline && + max_child_complexity <= current_opts_->max_compact_array_complexity); + if (wants_table_columns) { + metrics.is_uniform_array = check_array_uniformity(arr, metrics, depth); } return metrics; } inline element_metrics structure_analyzer::analyze_object(const dom::object& obj, - size_t depth) { + size_t depth) const { element_metrics metrics; metrics.complexity = 1; metrics.estimated_inline_len = 2; // "{}" @@ -11447,16 +12694,15 @@ inline element_metrics structure_analyzer::analyze_object(const dom::object& obj metrics.complexity = 1 + max_child_complexity; - metrics.can_inline = (metrics.complexity <= current_opts_->max_inline_complexity) && - (metrics.estimated_inline_len <= current_opts_->max_inline_length); - - // Objects use inline or expanded (no table/compact for objects) - if (metrics.child_count == 0 || metrics.can_inline) { - metrics.recommended_layout = layout_mode::INLINE; - } else { - metrics.recommended_layout = layout_mode::EXPANDED; + // Bracket padding '{ "a": 1 }' vs '{"a": 1}' + bool use_bracket_padding = (max_child_complexity >= 1) + ? current_opts_->nested_bracket_padding : current_opts_->simple_bracket_padding; + if (use_bracket_padding && metrics.child_count > 0) { + metrics.estimated_inline_len += 2; } + metrics.can_inline = metrics.complexity <= current_opts_->max_inline_complexity; + return metrics; } @@ -11473,8 +12719,18 @@ inline size_t structure_analyzer::estimate_string_length(std::string_view s) con } inline size_t structure_analyzer::estimate_number_length(double d) const { - if (std::isnan(d) || std::isinf(d)) { + if (!std::isfinite(d)) { +#if SIMDJSON_ENABLE_NAN_INF + if (std::isnan(d)) { + return 3; // "NaN" + } else if (d < 0) { + return 9; // "-Infinity" + } else { + return 8; // "Infinity" + } +#else return 4; // "null" for invalid numbers +#endif } // Rough estimate: up to 17 significant digits + sign + decimal point + exponent char buf[32]; @@ -11505,115 +12761,275 @@ inline size_t structure_analyzer::estimate_number_length(uint64_t u) const { return len; } -inline bool structure_analyzer::check_array_uniformity(const dom::array& arr, - std::vector& common_keys) const { - common_keys.clear(); - - std::set shared_keys; - dom::object first_obj; - bool have_first = false; - size_t object_count = 0; +inline table_column_type structure_analyzer::classify_table_value(dom::element_type type) { + switch (type) { + case dom::element_type::OBJECT: return table_column_type::object; + case dom::element_type::ARRAY: return table_column_type::array; + case dom::element_type::INT64: + case dom::element_type::UINT64: + case dom::element_type::DOUBLE: return table_column_type::number; + case dom::element_type::NULL_VALUE: return table_column_type::unknown; + default: return table_column_type::simple; // string, bool + } +} - for (dom::element elem : arr) { - if (elem.type() != dom::element_type::OBJECT) { - return false; // Not all elements are objects +inline void structure_analyzer::classify_and_measure( + const std::vector>& values, + table_column_type& common, size_t& max_width) { + common = table_column_type::unknown; + max_width = 0; + for (const auto& v : values) { + table_column_type t = classify_table_value(v.first.type()); + if (t != table_column_type::unknown) { + if (common == table_column_type::unknown) common = t; + else if (t != common) common = table_column_type::mixed; } - - dom::object obj; - if (elem.get_object().get(obj) != SUCCESS) { - return false; + if (v.second) { + max_width = (std::max)(max_width, v.second->estimated_inline_len); } + } +} - std::set current_keys; - for (dom::key_value_pair field : obj) { - current_keys.insert(std::string(field.key)); +inline void structure_analyzer::build_table_columns( + const std::vector>& values, + std::vector& out_columns) const { + out_columns.clear(); + if (values.empty()) { + return; + } + + table_column_type common = table_column_type::unknown; + for (const auto& v : values) { + table_column_type t = classify_table_value(v.first.type()); + if (t == table_column_type::unknown) continue; + if (common == table_column_type::unknown) common = t; + else if (t != common) { common = table_column_type::mixed; break; } + } + if (common != table_column_type::object && common != table_column_type::array) { + return; + } + + std::vector>> per_column_values; + + if (common == table_column_type::object) { + std::unordered_map column_index; + for (const auto& v : values) { + if (v.first.type() != dom::element_type::OBJECT) continue; + dom::object obj; + if (v.first.get_object().get(obj) != SUCCESS) continue; + + size_t field_idx = 0; + for (dom::key_value_pair field : obj) { + auto it = column_index.find(field.key); + size_t col_idx; + if (it == column_index.end()) { + col_idx = out_columns.size(); + column_index.emplace(field.key, col_idx); + out_columns.emplace_back(); + out_columns.back().key.assign(field.key.data(), field.key.size()); + out_columns.back().key_width = estimate_string_length(field.key); + per_column_values.emplace_back(); + } else { + col_idx = it->second; + } + const element_metrics* field_metrics = (v.second && field_idx < v.second->children.size()) + ? &v.second->children[field_idx] : nullptr; + per_column_values[col_idx].emplace_back(field.value, field_metrics); + field_idx++; + } } + } else { // array: columns by position + for (const auto& v : values) { + if (v.first.type() != dom::element_type::ARRAY) continue; + dom::array sub_arr; + if (v.first.get_array().get(sub_arr) != SUCCESS) continue; - if (!have_first) { - shared_keys = current_keys; - first_obj = obj; - have_first = true; - } else { - // Check similarity threshold against the first object - double similarity = compute_object_similarity(first_obj, obj); - if (similarity < current_opts_->table_similarity_threshold) { - return false; // Objects are too dissimilar for table format + size_t idx = 0; + for (dom::element item : sub_arr) { + if (out_columns.size() <= idx) { + out_columns.emplace_back(); + per_column_values.emplace_back(); + } + const element_metrics* item_metrics = (v.second && idx < v.second->children.size()) + ? &v.second->children[idx] : nullptr; + per_column_values[idx].emplace_back(item, item_metrics); + idx++; } + } + } + + for (size_t i = 0; i < out_columns.size(); i++) { + table_column_type col_type; + size_t max_width; + classify_and_measure(per_column_values[i], col_type, max_width); + out_columns[i].type = col_type; + out_columns[i].plain_width = max_width; - // Intersect with current keys - std::set intersection; - std::set_intersection(shared_keys.begin(), shared_keys.end(), - current_keys.begin(), current_keys.end(), - std::inserter(intersection, intersection.begin())); - shared_keys = intersection; + if (col_type == table_column_type::object || col_type == table_column_type::array) { + build_table_columns(per_column_values[i], out_columns[i].children); } - object_count++; + out_columns[i].width = out_columns[i].children.empty() ? max_width : compute_columns_width(out_columns[i].children); } +} - if (object_count < current_opts_->min_table_rows) { - return false; +inline bool structure_analyzer::check_array_uniformity(const dom::array& arr, + element_metrics& metrics, + size_t depth) const { + std::vector> values; + values.reserve(metrics.child_count); + + size_t row_idx = 0; + for (dom::element elem : arr) { + const element_metrics* row_metrics = (row_idx < metrics.children.size()) ? &metrics.children[row_idx] : nullptr; + values.emplace_back(elem, row_metrics); + row_idx++; + } + + build_table_columns(values, metrics.table_columns); + if (metrics.table_columns.empty()) { + // Not uniformly object or array. Check for uniform scalar + table_column_type common; + size_t max_width; + classify_and_measure(values, common, max_width); + if (common != table_column_type::number && common != table_column_type::simple) { + return false; + } + metrics.scalar_column_type = common; + metrics.table_row_width = max_width; + metrics.table_row_width_full = max_width; + return true; } - // Require at least one common key for table formatting - if (shared_keys.empty()) { - return false; + metrics.table_row_width_full = compute_columns_width(metrics.table_columns); + + size_t row_indent_width = (depth + 1) * current_opts_->indent_spaces; + size_t budget = (row_indent_width + 1 >= current_opts_->max_total_line_length) + ? 0 : current_opts_->max_total_line_length - row_indent_width - 1; + + size_t width = metrics.table_row_width_full; + while (width > budget && flatten_deepest_columns(metrics.table_columns)) { + recompute_column_widths(metrics.table_columns); + width = compute_columns_width(metrics.table_columns); } - common_keys.assign(shared_keys.begin(), shared_keys.end()); + metrics.table_row_width = width; return true; } -inline double structure_analyzer::compute_object_similarity(const dom::object& a, - const dom::object& b) const { - std::set keys_a, keys_b; - for (dom::key_value_pair field : a) { - keys_a.insert(std::string(field.key)); +inline size_t structure_analyzer::compute_columns_width(const std::vector& columns) const { + size_t width = 2; // "{}" or "[]" + if (table_row_is_nested(columns) ? current_opts_->nested_bracket_padding : current_opts_->simple_bracket_padding) { + width += 2; } - for (dom::key_value_pair field : b) { - keys_b.insert(std::string(field.key)); + + for (const table_column& col : columns) { + if (!col.key.empty()) { + width += col.key_width; + width += current_opts_->colon_padding ? 2 : 1; + } + width += col.width; + } + if (columns.size() > 1) { + width += (columns.size() - 1) * (current_opts_->comma_padding ? 2 : 1); + } + return width; +} + +inline size_t structure_analyzer::column_height(const table_column& column) { + size_t height = 0; + for (const table_column& child : column.children) { + height = (std::max)(height, column_height(child)); } + return column.children.empty() ? 0 : height + 1; +} - std::set intersection; - std::set_intersection(keys_a.begin(), keys_a.end(), - keys_b.begin(), keys_b.end(), - std::inserter(intersection, intersection.begin())); +inline bool structure_analyzer::flatten_deepest_columns(std::vector& columns) { + size_t max_height = 0; + for (const table_column& col : columns) { + max_height = (std::max)(max_height, column_height(col)); + } - std::set union_set; - std::set_union(keys_a.begin(), keys_a.end(), - keys_b.begin(), keys_b.end(), - std::inserter(union_set, union_set.begin())); + bool changed = false; + for (table_column& col : columns) { + if (column_height(col) != max_height || max_height == 0) continue; + if (max_height == 1) { + col.children.clear(); + col.width = col.plain_width; + changed = true; + } else { + changed |= flatten_deepest_columns(col.children); + } + } + return changed; +} - if (union_set.empty()) return 1.0; - return static_cast(intersection.size()) / static_cast(union_set.size()); +inline void structure_analyzer::recompute_column_widths(std::vector& columns) const { + for (table_column& col : columns) { + if (!col.children.empty()) { + recompute_column_widths(col.children); + col.width = compute_columns_width(col.children); + } + } } inline layout_mode structure_analyzer::decide_layout(const element_metrics& metrics, size_t depth, - size_t available_width) const { + const fractured_json_options& opts, + bool has_trailing_comma) { if (metrics.child_count == 0) { - return layout_mode::INLINE; + return layout_mode::single_line; } + long long signed_depth = static_cast(depth); + bool depth_allows_inline_or_compact = signed_depth > opts.always_expand_depth; + bool depth_allows_table = signed_depth >= opts.always_expand_depth; + // Check inline feasibility - size_t indent_width = depth * current_opts_->indent_spaces; - if (metrics.can_inline && - metrics.estimated_inline_len + indent_width <= available_width) { - return layout_mode::INLINE; + size_t reserved_width = depth * opts.indent_spaces + (has_trailing_comma ? 1 : 0); + if (depth_allows_inline_or_compact && metrics.can_inline && + metrics.estimated_inline_len + reserved_width <= opts.max_total_line_length) { + return layout_mode::single_line; } - // Check table mode - if (metrics.is_uniform_array && !metrics.common_keys.empty()) { - return layout_mode::TABLE; - } + // Rows (table's or compact multiline's) render one level deeper than the + // array itself. + size_t row_indent_width = (depth + 1) * opts.indent_spaces; // Check compact multiline - if (current_opts_->enable_compact_multiline && - metrics.complexity <= current_opts_->max_compact_array_complexity + 1) { - return layout_mode::COMPACT_MULTILINE; + // for uniform arrays fall back to table if we would have to flatten any formatting + bool compact_multiline_enabled = opts.enable_compact_multiline && + metrics.complexity <= opts.max_compact_array_complexity + 1 && + metrics.child_count >= opts.min_compact_array_row_items; + if (depth_allows_inline_or_compact && compact_multiline_enabled) { + bool aligned = metrics.is_uniform_array; + size_t comma_width = opts.comma_padding ? 2 : 1; + size_t avg_item_width; + if (aligned) { + avg_item_width = metrics.table_row_width_full + comma_width; + } else { + size_t sum = 0; + for (const element_metrics& child : metrics.children) { + sum += child.estimated_inline_len; + } + avg_item_width = comma_width + sum / metrics.child_count; + } + + size_t row_pack_space = (row_indent_width >= opts.max_total_line_length) + ? 0 : opts.max_total_line_length - row_indent_width; + if (avg_item_width * opts.min_compact_array_row_items <= row_pack_space) { + return layout_mode::compact_multiline; + } + } + + // Check Table mode + if (depth_allows_table && opts.enable_table_format && + metrics.is_uniform_array && + metrics.table_row_width + 1 + row_indent_width <= opts.max_total_line_length) { + return layout_mode::table; } - return layout_mode::EXPANDED; + return layout_mode::expanded; } // @@ -11621,10 +13037,10 @@ inline layout_mode structure_analyzer::decide_layout(const element_metrics& metr // inline fractured_formatter::fractured_formatter(const fractured_json_options& opts) - : options_(opts), column_widths_{} {} + : options_(opts) {} simdjson_inline void fractured_formatter::print_newline() { - if (current_layout_ == layout_mode::INLINE) { + if (current_layout_ == layout_mode::single_line) { return; // No newlines in inline mode } one_char('\n'); @@ -11632,7 +13048,7 @@ simdjson_inline void fractured_formatter::print_newline() { } simdjson_inline void fractured_formatter::print_indents(size_t depth) { - if (current_layout_ == layout_mode::INLINE) { + if (current_layout_ == layout_mode::single_line) { return; // No indentation in inline mode } for (size_t i = 0; i < depth * options_.indent_spaces; i++) { @@ -11654,26 +13070,10 @@ inline layout_mode fractured_formatter::get_layout_mode() const { return current_layout_; } -inline void fractured_formatter::set_depth(size_t depth) { - current_depth_ = depth; -} - -inline size_t fractured_formatter::get_depth() const { - return current_depth_; -} - inline void fractured_formatter::track_line_length(size_t chars) { current_line_length_ += chars; } -inline void fractured_formatter::reset_line_length() { - current_line_length_ = 0; -} - -inline size_t fractured_formatter::get_line_length() const { - return current_line_length_; -} - inline bool fractured_formatter::should_break_line(size_t upcoming_length) const { return (current_line_length_ + upcoming_length) > options_.max_total_line_length; } @@ -11682,39 +13082,6 @@ inline const fractured_json_options& fractured_formatter::options() const { return options_; } -inline void fractured_formatter::begin_table_row() { - in_table_mode_ = true; - current_column_ = 0; -} - -inline void fractured_formatter::end_table_row() { - in_table_mode_ = false; - current_column_ = 0; -} - -inline void fractured_formatter::set_column_widths(const std::vector& widths) { - column_widths_ = widths; -} - -inline size_t fractured_formatter::get_column_index() const { - return current_column_; -} - -inline void fractured_formatter::next_column() { - current_column_++; -} - -inline void fractured_formatter::align_to_column_width(size_t actual_width) { - if (current_column_ < column_widths_.size()) { - size_t target_width = column_widths_[current_column_]; - while (actual_width < target_width) { - one_char(' '); - actual_width++; - current_line_length_++; - } - } -} - // // Fractured String Builder Implementation // @@ -11753,19 +13120,20 @@ simdjson_inline std::string_view fractured_string_builder::str() const { inline void fractured_string_builder::format_element(const dom::element& elem, const element_metrics& metrics, - size_t depth) { + size_t depth, + bool has_trailing_comma) { switch (elem.type()) { case dom::element_type::ARRAY: { dom::array arr; if (elem.get_array().get(arr) == SUCCESS) { - format_array(arr, metrics, depth); + format_array(arr, metrics, depth, has_trailing_comma); } break; } case dom::element_type::OBJECT: { dom::object obj; if (elem.get_object().get(obj) == SUCCESS) { - format_object(obj, metrics, depth); + format_object(obj, metrics, depth, has_trailing_comma); } break; } @@ -11777,18 +13145,20 @@ inline void fractured_string_builder::format_element(const dom::element& elem, inline void fractured_string_builder::format_array(const dom::array& arr, const element_metrics& metrics, - size_t depth) { - switch (metrics.recommended_layout) { - case layout_mode::INLINE: + size_t depth, + bool has_trailing_comma) { + layout_mode layout = structure_analyzer::decide_layout(metrics, depth, options_, has_trailing_comma); + switch (layout) { + case layout_mode::single_line: format_array_inline(arr, metrics); break; - case layout_mode::COMPACT_MULTILINE: + case layout_mode::compact_multiline: format_array_compact_multiline(arr, metrics, depth); break; - case layout_mode::TABLE: + case layout_mode::table: format_array_as_table(arr, metrics, depth); break; - case layout_mode::EXPANDED: + case layout_mode::expanded: default: format_array_expanded(arr, metrics, depth); break; @@ -11797,8 +13167,7 @@ inline void fractured_string_builder::format_array(const dom::array& arr, inline void fractured_string_builder::format_array_inline(const dom::array& arr, const element_metrics& metrics) { - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); + scoped_single_line_mode single_line(format_); format_.start_array(); @@ -11812,60 +13181,69 @@ inline void fractured_string_builder::format_array_inline(const dom::array& arr, if (options_.comma_padding) { format_.print_space(); } - } else if (options_.simple_bracket_padding) { + } else if (bracket_padding_for(metrics)) { format_.print_space(); } first = false; - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; + const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); format_element(elem, child_metrics, 0); child_idx++; } - if (options_.simple_bracket_padding && !empty) { + if (bracket_padding_for(metrics) && !empty) { format_.print_space(); } format_.end_array(); - - format_.set_layout_mode(prev_layout); } inline void fractured_string_builder::format_array_compact_multiline(const dom::array& arr, const element_metrics& metrics, size_t depth) { + if (metrics.is_uniform_array) { + format_array_compact_multiline_aligned(arr, metrics, depth); + return; + } + format_.start_array(); format_.print_newline(); format_.print_indents(depth + 1); - size_t items_on_line = 0; bool first = true; + bool prev_item_was_expanded = false; size_t child_idx = 0; for (dom::element elem : arr) { + const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); + if (!first) { format_.comma(); + format_.track_line_length(1); // Check if we should break to new line - if (items_on_line >= options_.max_items_per_line || - format_.should_break_line(20)) { // 20 is rough estimate for next item + if (prev_item_was_expanded || + format_.should_break_line(child_metrics.estimated_inline_len)) { format_.print_newline(); format_.print_indents(depth + 1); - items_on_line = 0; } else if (options_.comma_padding) { format_.print_space(); } } first = false; - // Format element inline - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(elem, child_metrics, depth + 1); - format_.set_layout_mode(prev_layout); + bool is_last = (child_idx + 1 == metrics.child_count); + layout_mode item_layout = structure_analyzer::decide_layout(child_metrics, depth + 1, options_, !is_last); + bool item_fits = item_layout == layout_mode::single_line; + if (item_fits) { + { + scoped_single_line_mode single_line(format_); + format_element(elem, child_metrics, depth + 1, !is_last); + } + format_.track_line_length(child_metrics.estimated_inline_len); + } else { + format_element(elem, child_metrics, depth + 1, !is_last); + } + prev_item_was_expanded = !item_fits; - items_on_line++; child_idx++; } @@ -11874,114 +13252,309 @@ inline void fractured_string_builder::format_array_compact_multiline(const dom:: format_.end_array(); } -inline void fractured_string_builder::format_array_as_table(const dom::array& arr, - const element_metrics& metrics, - size_t depth) { - const std::vector& columns = metrics.common_keys; - if (columns.empty()) { - format_array_expanded(arr, metrics, depth); - return; - } - - // Calculate column widths for alignment - std::vector col_widths = calculate_column_widths(arr, columns); - format_.set_column_widths(col_widths); +inline void fractured_string_builder::format_array_compact_multiline_aligned( + const dom::array& arr, const element_metrics& metrics, size_t depth) { + const std::vector& columns = metrics.table_columns; format_.start_array(); format_.print_newline(); + format_.print_indents(depth + 1); - bool first_row = true; + size_t indent_width = (depth + 1) * options_.indent_spaces; + size_t available_line_space = (indent_width >= options_.max_total_line_length) + ? 0 : options_.max_total_line_length - indent_width; + size_t comma_width = options_.comma_padding ? 2 : 1; + size_t remaining_line_space = available_line_space; + + bool first = true; size_t child_idx = 0; + for (dom::element elem : arr) { - if (!first_row) { - format_.comma(); - format_.print_newline(); - } - first_row = false; + bool needs_comma = (child_idx + 1 < metrics.child_count); + size_t space_needed = metrics.table_row_width_full + (needs_comma ? comma_width : 0); - format_.print_indents(depth + 1); - format_.begin_table_row(); + if (!first) { + if (remaining_line_space < space_needed) { + format_.print_newline(); + format_.print_indents(depth + 1); + remaining_line_space = available_line_space; + } else if (options_.comma_padding) { + format_.print_space(); + } + } + first = false; - // Format object as inline with aligned columns - dom::object obj; - if (elem.get_object().get(obj) != SUCCESS) { - child_idx++; - continue; + const element_metrics& row_metrics = child_metrics_at(metrics.children, child_idx); + if (columns.empty()) { + format_table_scalar_row(elem, row_metrics, metrics.table_row_width_full, depth + 1, + metrics.scalar_column_type); + } else { + format_table_row(elem, row_metrics, columns, depth + 1); } + if (needs_comma) { + format_.comma(); + } + remaining_line_space -= (std::min)(remaining_line_space, space_needed); + child_idx++; + } - // Get child metrics for this row (object) - const element_metrics& row_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; + format_.print_newline(); + format_.print_indents(depth); + format_.end_array(); +} - format_.start_object(); - if (options_.simple_bracket_padding) { - format_.print_space(); - } +inline void fractured_string_builder::format_table_row_columns( + const std::vector& columns, + const std::vector& found, + const std::vector& values, + const std::vector& value_metrics, + size_t depth) { + const size_t num_columns = columns.size(); + size_t last_present_idx = num_columns; + for (size_t i = 0; i < num_columns; i++) { + if (found[i]) last_present_idx = i; + } - bool first_col = true; - const size_t num_columns = columns.size(); + size_t comma_width = options_.comma_padding ? 2 : 1; - for (size_t col_idx = 0; col_idx < num_columns; col_idx++) { - const std::string& key = columns[col_idx]; - const bool is_last_col = (col_idx == num_columns - 1); + for (size_t col_idx = 0; col_idx < num_columns; col_idx++) { + const table_column& column = columns[col_idx]; + const bool is_last_col = (col_idx == num_columns - 1); - if (!first_col) { - format_.comma(); - if (options_.comma_padding) { + if (found[col_idx]) { + if (!column.key.empty()) { + format_.key(column.key); + if (options_.colon_padding) { format_.print_space(); } } - first_col = false; - // Write key - format_.key(key); - if (options_.colon_padding) { - format_.print_space(); - } + bool needs_comma = !is_last_col && (col_idx < last_present_idx); - // Find the value for this key and its metrics - dom::element value; - bool found = false; - size_t field_idx = 0; - for (dom::key_value_pair field : obj) { - if (field.key == key) { - value = field.value; - found = true; - break; + if (!column.children.empty()) { + // Recurses into this cell's own columns instead of a plain value; + // every row aligns those the same way (blank-padding missing + // ones), so the result is always exactly column.width wide + // no padding needed afterward, unlike the leaf case below. + if (column.type == table_column_type::object) { + dom::object sub_obj; + if (values[col_idx].get_object().get(sub_obj) == SUCCESS) { + const element_metrics& sub_metrics = child_metrics_at(value_metrics[col_idx]); + format_table_object_row(sub_obj, sub_metrics, column.children, depth); + } + } else { + dom::array sub_arr; + if (values[col_idx].get_array().get(sub_arr) == SUCCESS) { + const element_metrics& sub_metrics = child_metrics_at(value_metrics[col_idx]); + format_table_array_row(sub_arr, sub_metrics, column.children, depth); + } } - field_idx++; + // value is already padded + if (needs_comma) { + format_.comma(); + if (options_.comma_padding) { + format_.print_space(); + } + } + } else { + const element_metrics& vm = child_metrics_at(value_metrics[col_idx]); + format_table_leaf_value(values[col_idx], vm, column.width, column.type, needs_comma, + /*add_comma_space=*/true, depth); } - // Write value - if (found) { - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); - const element_metrics& value_metrics = (field_idx < row_metrics.children.size()) - ? row_metrics.children[field_idx] : element_metrics{}; - format_element(value, value_metrics, depth + 1); - format_.set_layout_mode(prev_layout); - } else { - format_.null_atom(); + if (!is_last_col && !needs_comma) { + // Found, but no more real values follow: blank space where a comma would go. + for (size_t i = 0; i < comma_width; i++) { + format_.one_char(' '); + } + } + } else { + size_t slot_width = column.width; + if (!column.key.empty()) { + slot_width += column.key_width + (options_.colon_padding ? 2 : 1); + } + for (size_t i = 0; i < slot_width; i++) { + format_.one_char(' '); } - // Only pad non-last columns to align values across rows if (!is_last_col) { - size_t actual_len = found ? measure_value_length(value) : 4; // 4 for "null" - size_t target_width = col_widths[col_idx]; - while (actual_len < target_width) { + for (size_t i = 0; i < comma_width; i++) { format_.one_char(' '); - actual_len++; } } + } + } +} - format_.next_column(); +inline void fractured_string_builder::format_table_object_row( + const dom::object& obj, const element_metrics& row_metrics, + const std::vector& columns, size_t depth) { + const size_t num_columns = columns.size(); + std::vector found(num_columns, false); + std::vector values(num_columns); + std::vector value_metrics(num_columns, nullptr); + + for (size_t col_idx = 0; col_idx < num_columns; col_idx++) { + size_t field_idx = 0; + for (dom::key_value_pair field : obj) { + if (field.key == columns[col_idx].key) { + found[col_idx] = true; + values[col_idx] = field.value; + value_metrics[col_idx] = (field_idx < row_metrics.children.size()) + ? &row_metrics.children[field_idx] : nullptr; + break; + } + field_idx++; } + } - if (options_.simple_bracket_padding) { - format_.print_space(); + bool nested = table_row_is_nested(columns); + format_.start_object(); + if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { + format_.print_space(); + } + format_table_row_columns(columns, found, values, value_metrics, depth); + if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { + format_.print_space(); + } + format_.end_object(); +} + +inline void fractured_string_builder::format_table_array_row( + const dom::array& arr, const element_metrics& row_metrics, + const std::vector& columns, size_t depth) { + const size_t num_columns = columns.size(); + std::vector found(num_columns, false); + std::vector values(num_columns); + std::vector value_metrics(num_columns, nullptr); + + size_t idx = 0; + for (dom::element item : arr) { + if (idx >= num_columns) break; + found[idx] = true; + values[idx] = item; + value_metrics[idx] = (idx < row_metrics.children.size()) ? &row_metrics.children[idx] : nullptr; + idx++; + } + + bool nested = table_row_is_nested(columns); + format_.start_array(); + if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { + format_.print_space(); + } + format_table_row_columns(columns, found, values, value_metrics, depth); + if (nested ? options_.nested_bracket_padding : options_.simple_bracket_padding) { + format_.print_space(); + } + format_.end_array(); +} + +inline void fractured_string_builder::format_table_row( + const dom::element& elem, const element_metrics& row_metrics, + const std::vector& columns, size_t depth) { + if (elem.type() == dom::element_type::ARRAY) { + dom::array arr; + if (elem.get_array().get(arr) == SUCCESS) { + format_table_array_row(arr, row_metrics, columns, depth); + } + } else { + dom::object obj; + if (elem.get_object().get(obj) == SUCCESS) { + format_table_object_row(obj, row_metrics, columns, depth); + } + } +} + +inline bool fractured_string_builder::comma_goes_before_padding(table_column_type column_type) const { + switch (options_.comma_placement) { + case table_comma_placement::before_padding: return true; + case table_comma_placement::after_padding: return false; + case table_comma_placement::before_padding_except_numbers: + default: + return column_type != table_column_type::number; + } +} + +inline void fractured_string_builder::format_table_leaf_value( + const dom::element& elem, const element_metrics& vm, size_t width, + table_column_type column_type, bool needs_comma, bool add_comma_space, size_t depth) { + bool comma_before_pad = needs_comma && comma_goes_before_padding(column_type); + bool comma_after_pad = needs_comma && !comma_before_pad; + + bool right_align = column_type == table_column_type::number && + options_.number_alignment == number_list_alignment::right; + + size_t value_len = vm.estimated_inline_len; + size_t left_pad = 0; + size_t right_pad = 0; + if (right_align) { + left_pad = (width > value_len) ? width - value_len : 0; + comma_before_pad = needs_comma; + comma_after_pad = false; + } else { + right_pad = (width > value_len) ? width - value_len : 0; + } + + for (size_t i = 0; i < left_pad; i++) { + format_.one_char(' '); + } + + { + scoped_single_line_mode single_line(format_); + format_element(elem, vm, depth); + } + + if (comma_before_pad) { + format_.comma(); + } + for (size_t i = 0; i < right_pad; i++) { + format_.one_char(' '); + } + if (comma_after_pad) { + format_.comma(); + } + if (needs_comma && add_comma_space && options_.comma_padding) { + format_.print_space(); + } +} + +inline void fractured_string_builder::format_table_scalar_row( + const dom::element& elem, const element_metrics& row_metrics, size_t width, size_t depth, + table_column_type column_type) { + format_table_leaf_value(elem, row_metrics, width, column_type, + /*needs_comma=*/false, /*add_comma_space=*/false, depth); +} + +inline void fractured_string_builder::format_array_as_table(const dom::array& arr, + const element_metrics& metrics, + size_t depth) { + if (!metrics.is_uniform_array) { + format_array_expanded(arr, metrics, depth); + return; + } + const std::vector& columns = metrics.table_columns; + + format_.start_array(); + format_.print_newline(); + + bool first_row = true; + size_t child_idx = 0; + for (dom::element elem : arr) { + if (!first_row) { + format_.comma(); + format_.print_newline(); + } + first_row = false; + + format_.print_indents(depth + 1); + + const element_metrics& row_metrics = child_metrics_at(metrics.children, child_idx); + if (columns.empty()) { + format_table_scalar_row(elem, row_metrics, metrics.table_row_width, depth + 1, + metrics.scalar_column_type); + } else { + format_table_row(elem, row_metrics, columns, depth + 1); } - format_.end_object(); - format_.end_table_row(); child_idx++; } @@ -12008,9 +13581,9 @@ inline void fractured_string_builder::format_array_expanded(const dom::array& ar format_.print_newline(); format_.print_indents(depth + 1); - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(elem, child_metrics, depth + 1); + const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); + bool is_last = (child_idx + 1 == metrics.child_count); + format_element(elem, child_metrics, depth + 1, !is_last); child_idx++; } @@ -12023,8 +13596,10 @@ inline void fractured_string_builder::format_array_expanded(const dom::array& ar inline void fractured_string_builder::format_object(const dom::object& obj, const element_metrics& metrics, - size_t depth) { - if (metrics.recommended_layout == layout_mode::INLINE || metrics.can_inline) { + size_t depth, + bool has_trailing_comma) { + layout_mode layout = structure_analyzer::decide_layout(metrics, depth, options_, has_trailing_comma); + if (layout == layout_mode::single_line) { format_object_inline(obj, metrics); } else { format_object_expanded(obj, metrics, depth); @@ -12033,8 +13608,7 @@ inline void fractured_string_builder::format_object(const dom::object& obj, inline void fractured_string_builder::format_object_inline(const dom::object& obj, const element_metrics& metrics) { - layout_mode prev_layout = format_.get_layout_mode(); - format_.set_layout_mode(layout_mode::INLINE); + scoped_single_line_mode single_line(format_); format_.start_object(); @@ -12049,7 +13623,7 @@ inline void fractured_string_builder::format_object_inline(const dom::object& ob if (options_.comma_padding) { format_.print_space(); } - } else if (options_.simple_bracket_padding) { + } else if (bracket_padding_for(metrics)) { format_.print_space(); } first = false; @@ -12058,18 +13632,15 @@ inline void fractured_string_builder::format_object_inline(const dom::object& ob if (options_.colon_padding) { format_.print_space(); } - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; + const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); format_element(field.value, child_metrics, 0); child_idx++; } - if (options_.simple_bracket_padding && !empty) { + if (bracket_padding_for(metrics) && !empty) { format_.print_space(); } format_.end_object(); - - format_.set_layout_mode(prev_layout); } inline void fractured_string_builder::format_object_expanded(const dom::object& obj, @@ -12094,9 +13665,9 @@ inline void fractured_string_builder::format_object_expanded(const dom::object& if (options_.colon_padding) { format_.print_space(); } - const element_metrics& child_metrics = (child_idx < metrics.children.size()) - ? metrics.children[child_idx] : element_metrics{}; - format_element(field.value, child_metrics, depth + 1); + const element_metrics& child_metrics = child_metrics_at(metrics.children, child_idx); + bool is_last = (child_idx + 1 == metrics.child_count); + format_element(field.value, child_metrics, depth + 1, !is_last); child_idx++; } @@ -12152,97 +13723,8 @@ inline void fractured_string_builder::format_scalar(const dom::element& elem) { } } -inline size_t fractured_string_builder::measure_value_length(const dom::element& elem) const { - switch (elem.type()) { - case dom::element_type::STRING: { - std::string_view str; - if (elem.get_string().get(str) == SUCCESS) { - // Count actual escaped length - size_t len = 2; // quotes - for (char c : str) { - if (c == '"' || c == '\\' || static_cast(c) < 32) { - len += 2; // escape sequence - } else { - len += 1; - } - } - return len; - } - return 2; - } - case dom::element_type::INT64: { - int64_t val; - if (elem.get_int64().get(val) == SUCCESS) { - if (val == 0) return 1; - // Handle INT64_MIN specially to avoid overflow when negating - if (val == INT64_MIN) return 20; // "-9223372036854775808" is 20 characters - size_t len = (val < 0) ? 1 : 0; - int64_t abs_val = (val < 0) ? -val : val; - while (abs_val > 0) { len++; abs_val /= 10; } - return len; - } - return 1; - } - case dom::element_type::UINT64: { - uint64_t val; - if (elem.get_uint64().get(val) == SUCCESS) { - if (val == 0) return 1; - size_t len = 0; - while (val > 0) { len++; val /= 10; } - return len; - } - return 1; - } - case dom::element_type::DOUBLE: { - double val; - if (elem.get_double().get(val) == SUCCESS) { - char buf[32]; - int len = snprintf(buf, sizeof(buf), "%.17g", val); - return len > 0 ? static_cast(len) : 1; - } - return 1; - } - case dom::element_type::BOOL: { - bool val; - if (elem.get_bool().get(val) == SUCCESS) { - return val ? 4 : 5; // "true" or "false" - } - return 5; - } - case dom::element_type::NULL_VALUE: - return 4; // "null" - default: - return 4; - } -} - -inline std::vector fractured_string_builder::calculate_column_widths( - const dom::array& arr, - const std::vector& columns) const { - - std::vector widths(columns.size(), 0); - - for (dom::element elem : arr) { - dom::object obj; - if (elem.get_object().get(obj) != SUCCESS) { - continue; - } - - for (size_t col_idx = 0; col_idx < columns.size(); col_idx++) { - const std::string& key = columns[col_idx]; - - for (dom::key_value_pair field : obj) { - if (field.key == key) { - // Measure actual value length - size_t len = measure_value_length(field.value); - widths[col_idx] = (std::max)(widths[col_idx], len); - break; - } - } - } - } - - return widths; +inline bool fractured_string_builder::bracket_padding_for(const element_metrics& metrics) const { + return metrics.complexity >= 2 ? options_.nested_bracket_padding : options_.simple_bracket_padding; } } // namespace internal @@ -12282,19 +13764,6 @@ std::string fractured_json(simdjson_result x, const fractured_json_options& o } #endif -// Explicit template instantiations for common types -template std::string fractured_json(dom::element x); -template std::string fractured_json(dom::element x, const fractured_json_options& options); -template std::string fractured_json(dom::array x); -template std::string fractured_json(dom::array x, const fractured_json_options& options); -template std::string fractured_json(dom::object x); -template std::string fractured_json(dom::object x, const fractured_json_options& options); - -#if SIMDJSON_EXCEPTIONS -template std::string fractured_json(simdjson_result x); -template std::string fractured_json(simdjson_result x, const fractured_json_options& options); -#endif - // // String-based API for formatting any JSON string // @@ -12662,6 +14131,8 @@ enum instruction_set { LASX = 0x40000, //RVV = 0x80000, RVV_VLS = 0x100000, + SVE = 0x200000, + SVE2 = 0x400000, }; } // namespace internal @@ -13774,7 +15245,7 @@ class implementation final : public simdjson::implementation { simdjson_inline implementation() : simdjson::implementation( "rvv_vls", "RISC-V V extension", - 0 + internal::instruction_set::RVV_VLS ) {} simdjson_warn_unused error_code create_dom_parser_implementation( size_t capacity, @@ -13933,7 +15404,14 @@ simdjson_inline int leading_zeroes(uint64_t input_num) { /* result might be undefined when input_num is zero */ simdjson_inline int count_ones(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); +#else + // if the system supports SVE or CSSC, __builtin_popcountll + // might be compiled to fewer single instructions. For CSSC, + // __builtin_popcountll is compiled to a single instruction. + return __builtin_popcountll(input_num); +#endif// SIMDJSON_REGULAR_VISUAL_STUDIO } @@ -13970,15 +15448,6 @@ simdjson_inline uint64_t zero_leading_bit(uint64_t rev_bits, int leading_zeroes) #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace arm64 @@ -14242,6 +15711,7 @@ namespace { return vget_lane_u64( vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(*this), 4)), 0); } + // Integer umaxv, not a float compare: flush-to-zero would treat a denormal as zero. simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; @@ -14890,6 +16360,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -14901,6 +16374,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -14937,6 +16432,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace arm64 @@ -15027,7 +16587,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -15206,6 +16766,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -15245,6 +16806,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -15501,6 +17074,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -15538,6 +17324,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -15556,6 +17352,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -15572,26 +17390,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -15680,7 +17541,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -15763,15 +17624,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -15802,7 +17665,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -15851,7 +17728,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -15950,7 +17827,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -16048,7 +17925,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -16103,7 +17980,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -16189,7 +18066,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -16229,11 +18106,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -16244,9 +18130,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -16295,67 +18180,12 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { // // Check for minus sign // @@ -16367,91 +18197,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + if ( p == src ) { - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; + return INCORRECT_TYPE; } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -16462,9 +18221,239 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { return INCORRECT_TYPE; } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t *start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -16513,6 +18502,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -17109,6 +19191,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -17120,6 +19205,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -17156,6 +19263,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace fallback @@ -17246,7 +19418,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -17425,6 +19597,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -17464,6 +19637,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -17720,6 +19905,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -17757,6 +20155,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -17775,6 +20183,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -17791,26 +20221,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -17899,7 +20372,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -17982,15 +20455,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -18021,7 +20496,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -18070,7 +20559,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -18169,7 +20658,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -18267,7 +20756,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -18322,7 +20811,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -18408,7 +20897,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -18448,11 +20937,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -18463,9 +20961,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -18514,6 +21011,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -18666,11 +21254,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -18681,9 +21283,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -18732,6 +21333,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -19053,16 +21747,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace haswell @@ -19815,6 +22499,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -19826,6 +22513,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -19862,6 +22571,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace haswell @@ -19952,7 +22726,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -20131,6 +22905,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -20170,6 +22945,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -20426,6 +23213,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -20463,6 +23463,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -20481,6 +23491,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -20497,26 +23529,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -20605,7 +23680,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -20688,15 +23763,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -20727,7 +23804,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -20776,7 +23867,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -20875,7 +23966,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -20973,7 +24064,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -21028,7 +24119,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -21114,7 +24205,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -21154,11 +24245,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -21169,9 +24269,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -21220,6 +24319,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -21372,11 +24562,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -21387,9 +24591,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -21438,6 +24641,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -21756,16 +25052,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace icelake @@ -22521,6 +25807,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -22532,6 +25821,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -22568,6 +25879,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace icelake @@ -22658,7 +26034,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -22837,6 +26213,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -22876,6 +26253,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -23132,6 +26521,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -23169,6 +26771,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -23187,6 +26799,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -23203,26 +26837,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -23311,7 +26988,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -23394,15 +27071,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -23433,7 +27112,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -23482,7 +27175,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -23581,7 +27274,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -23679,7 +27372,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -23734,7 +27427,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -23820,7 +27513,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -23860,11 +27553,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -23875,9 +27577,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -23926,6 +27627,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -24078,11 +27870,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -24093,9 +27899,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -24144,6 +27949,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -24434,16 +28332,6 @@ simdjson_inline int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - *result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace ppc64 @@ -25342,6 +29230,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -25353,6 +29244,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -25389,6 +29302,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace ppc64 @@ -25479,7 +29457,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -25658,6 +29636,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -25697,6 +29676,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -25953,6 +29944,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -25990,6 +30194,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -26008,6 +30222,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -26024,26 +30260,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -26132,7 +30411,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -26215,15 +30494,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -26254,7 +30535,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -26303,7 +30598,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -26402,7 +30697,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -26500,7 +30795,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -26555,7 +30850,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -26641,7 +30936,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -26681,11 +30976,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -26696,9 +31000,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -26747,67 +31050,12 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { // // Check for minus sign // @@ -26819,91 +31067,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + if ( p == src ) { - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; + return INCORRECT_TYPE; } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -26914,9 +31091,239 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { return INCORRECT_TYPE; } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t *start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -26965,6 +31372,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -27269,16 +31769,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -27857,16 +32347,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { } #endif -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif -} } // unnamed namespace } // namespace westmere @@ -28480,6 +32960,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -28491,6 +32974,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -28527,6 +33032,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace westmere @@ -28617,7 +33187,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -28796,6 +33366,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -28835,6 +33406,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -29091,6 +33674,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -29128,6 +33924,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -29146,6 +33952,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -29162,26 +33990,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -29270,7 +34141,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -29353,15 +34224,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -29392,7 +34265,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -29441,7 +34328,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -29540,7 +34427,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -29638,7 +34525,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -29693,7 +34580,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -29779,7 +34666,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -29819,11 +34706,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -29834,9 +34730,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -29885,6 +34780,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -30037,11 +35023,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -30052,9 +35052,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -30103,6 +35102,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -30368,10 +35460,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{input_num, 0, 0, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lasx @@ -31118,6 +36206,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -31129,6 +36220,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -31165,6 +36278,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace lasx @@ -31255,7 +36433,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -31434,6 +36612,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -31473,6 +36652,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -31729,6 +36920,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -31766,6 +37170,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -31784,6 +37198,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -31800,26 +37236,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -31908,7 +37387,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -31991,15 +37470,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -32030,7 +37511,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -32079,7 +37574,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -32178,7 +37673,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -32276,7 +37771,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -32331,7 +37826,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -32417,7 +37912,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -32457,11 +37952,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -32472,9 +37976,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -32523,6 +38026,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -32675,11 +38269,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -32690,9 +38298,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -32741,6 +38348,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -33002,10 +38702,6 @@ simdjson_inline int count_ones(uint64_t input_num) { return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{input_num, 0})), 0); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace lsx @@ -33734,6 +39430,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -33745,6 +39444,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -33781,6 +39502,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace lsx @@ -33871,7 +39657,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -34050,6 +39836,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -34089,6 +39876,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -34345,6 +40144,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -34382,6 +40394,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -34400,6 +40422,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -34416,26 +40460,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -34524,7 +40611,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -34607,15 +40694,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -34646,7 +40735,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -34695,7 +40798,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -34794,7 +40897,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -34892,7 +40995,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -34947,7 +41050,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -35033,7 +41136,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -35073,11 +41176,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -35088,9 +41200,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -35139,6 +41250,97 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return (*src == '-'); } @@ -35291,11 +41493,25 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -35306,9 +41522,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -35357,6 +41572,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -35622,11 +41930,6 @@ simdjson_inline long long int count_ones(uint64_t input_num) { return __builtin_popcountll(input_num); } -simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t *result) { - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -} } // unnamed namespace } // namespace rvv_vls @@ -36367,6 +42670,9 @@ namespace atomparsing { // to the compile-time constant 1936482662. simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } +// Acts on the same principle as string_to_uint32, but on an 8-byte block of memory +simdjson_inline uint64_t string_to_uint64(const char* str) { uint64_t val; std::memcpy(&val, str, sizeof(uint64_t)); return val; } + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. @@ -36378,6 +42684,28 @@ simdjson_inline uint32_t str4ncmp(const uint8_t *src, const char* atom) { return srcval ^ string_to_uint32(atom); } +// Checks that the first 8 characters of the input string match the given atom in a case-insensitive manner. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint64_t str8ncmp_case_insensitive(const uint8_t *src, const char* atom) { + uint64_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint64_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 8 bytes"); + std::memcpy(&srcval, src, sizeof(uint64_t)); + + return (srcval | 0x2020202020202020ull) ^ string_to_uint64(atom); +} + +// Checks that the first 3 characters of 'src' match 'atom' in a case-insensitive way. +// +// 'atom' must consist of only lowercase letters. +simdjson_warn_unused +simdjson_inline uint32_t str3ncmp_case_insensitive(const uint8_t *src, const char* atom) { + return ((src[0] | 0x20) ^ atom[0]) // + | ((src[1] | 0x20) ^ atom[1]) // + | ((src[2] | 0x20) ^ atom[2]); +} + simdjson_warn_unused simdjson_inline bool is_valid_true_atom(const uint8_t *src) { return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; @@ -36414,6 +42742,71 @@ simdjson_inline bool is_valid_null_atom(const uint8_t *src, size_t len) { else { return false; } } +#if SIMDJSON_ENABLE_NAN_INF +// "nan" is 3 bytes; we check characters and then verify the next +// character is structural or whitespace. We accept both "nan" and "NaN". +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; +} + +// checks that the next four characters of a string are 'nan"', where the 'nan' +// is checked in a case-insensitive way. +simdjson_warn_unused +simdjson_inline bool is_valid_nan_in_string(const uint8_t *src) { + return (str3ncmp_case_insensitive(src, "nan") | (src[3] ^ '"')) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_nan_atom(const uint8_t *src, size_t len) { + if (len > 3) { return is_valid_nan_atom(src); } + if (len == 3) { return str3ncmp_case_insensitive(src, "nan") == 0; } + return false; +} + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + if(is_short_inf) return true; + + // Check for 'infinity' (any capitalization) + return (str8ncmp_case_insensitive(src, "infinity") | jsoncharutils::is_not_structural_or_whitespace(src[8])) == 0; +} + +simdjson_warn_unused +simdjson_inline bool is_valid_inf_in_string(const uint8_t *src) { + bool is_short_inf = (str3ncmp_case_insensitive(src, "inf") | (src[3] ^ '"')) == 0; + if(is_short_inf) return true; + + return (str8ncmp_case_insensitive(src, "infinity") | (src[8] ^ '"')) == 0; +} + + +// This function will accept any case-insensitive 3-character spelling of +// infinity: 'inf', 'INF', and 'Inf' are all accepted. +// +// Any capitalization of 'infinity' is also accepted. +simdjson_warn_unused +simdjson_inline bool is_valid_inf_atom(const uint8_t *src, size_t len) { + if (len > 8) { return is_valid_inf_atom(src); } + if (len == 8 && str8ncmp_case_insensitive(src, "infinity") == 0) { + return true; + } + if (len > 3) { + return (str3ncmp_case_insensitive(src, "inf") + | jsoncharutils::is_not_structural_or_whitespace(src[3])) == 0; + } + if (len == 3) { return str3ncmp_case_insensitive(src, "inf") == 0; } + return false; +} +#endif // SIMDJSON_ENABLE_NAN_INF + } // namespace atomparsing } // unnamed namespace } // namespace rvv_vls @@ -36504,7 +42897,7 @@ inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(s } inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - if(max_depth == 0 || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } + if(max_depth == 0 || max_depth > SIMDJSON_MAX_DEPTH || max_depth > SIZE_MAX / sizeof(open_container)) { return CAPACITY; } // Stage 2 stacks open_containers.reset(new (std::nothrow) open_container[max_depth]); is_array.reset(new (std::nothrow) bool[max_depth]); @@ -36683,6 +43076,7 @@ struct implementation_simdjson_result_base { /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -36722,6 +43116,18 @@ simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool return d; } +// Convert a mantissa, an exponent and a sign bit into an ieee32 float (binary32). +// The real_exponent needs to be in [0, 254] (technically real_exponent = 255 would be acceptable). +// The mantissa should be in [0,1<<24). The bit at index (1U << 23) will be zeroed. +simdjson_inline float to_float(uint32_t mantissa, uint32_t real_exponent, bool negative) { + float f; + mantissa &= ~(uint32_t(1) << 23); + mantissa |= real_exponent << 23; + mantissa |= ((static_cast(negative)) << 31); + std::memcpy(&f, &mantissa, sizeof(f)); + return f; +} + // Attempts to compute i * 10^(power) exactly; and if "negative" is // true, negate the result. // This function will only work in some cases, when it does not work, success is @@ -36978,6 +43384,219 @@ simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, return true; } +// Attempts to compute i * 10^(power) as a binary32 (float) value; and if +// "negative" is true, negate the result. +// +// This is the same Eisel-Lemire algorithm as compute_float_64, retargeted at +// binary32. It is adapted from fast_float +// (https://github.com/fastfloat/fast_float), where the routine is written once +// and instantiated for each binary format. We only need 24 bits of mantissa +// instead of 53, so we ask the truncated product for 24 + 2 = 26 accurate bits +// (plus one bit that may be lost to the "upperbit" shift), and we shift the +// product right by (upperbit + 64 - 23 - 3) instead of (upperbit + 64 - 52 - 3). +// +// The power-of-five table is shared with the binary64 code: the useful range +// for binary32, [smallest_power_binary32, largest_power_binary32], is a strict +// subset of [smallest_power, largest_power]. +// +// The function returns false when the result would be infinite: simdjson +// refuses to parse infinite values, so the caller reports an error. Unlike +// compute_float_64, a false return never means "try harder": the accuracy of +// the two-limb product is guaranteed by Noble Mushtak and Daniel Lemire, +// "Fast Number Parsing Without Fallback" (https://arxiv.org/abs/2212.06644). +// The caller must still fall back to a slow path when the 64-bit mantissa i +// was truncated (more than 19 significant digits). +// +// We assume that power is in the [smallest_power, largest_power] interval: the +// caller is responsible for this check. +simdjson_inline bool compute_float_32(int64_t power, uint64_t i, bool negative, float &d) { + // Powers of ten that are exactly representable as binary32 values: 10^k is + // exact as long as k <= 10 (5^10 = 9765625 < 2**24 while 5^11 > 2**24). + static constexpr float power_of_ten_float[] = { + 1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, + 1e6f, 1e7f, 1e8f, 1e9f, 1e10f}; + // The range of powers of ten that a non-zero, finite binary32 value can be + // built from. Anything smaller rounds to zero, anything larger is infinite. + // These are the smallest_power_of_ten()/largest_power_of_ten() constants that + // fast_float uses for binary32. + constexpr int smallest_power_binary32 = -65; + constexpr int largest_power_binary32 = 38; + + // We start with the fast path described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 10 && i <= 16777215) +#else + if (-10 <= power && power <= 10 && i <= 16777215) +#endif + { + // Convert the integer into a float. This is lossless since + // 0 <= i <= 2^24 - 1. + d = float(i); + // Both d and the power of ten are exactly representable as binary32 + // values, so the product (or quotient) is correctly rounded. + if (power < 0) { + d = d / power_of_ten_float[-power]; + } else { + d = d * power_of_ten_float[power]; + } + if (negative) { + d = -d; + } + return true; + } + + // The fast path has failed, so we fall back on the Eisel-Lemire algorithm. + // It needs i > 0 (so that the leading bit of i can be normalized), so we + // handle i == 0 separately. We also handle the powers of ten that are so + // small (or so large) that the answer is zero (or infinite) whatever the + // mantissa is. + if (i == 0 || power < smallest_power_binary32) { + d = negative ? -0.0f : 0.0f; + return true; + } + if (power > largest_power_binary32) { + // We have, for sure, an infinite value. + return false; + } + + // The exponent is 128 + 63 + power + floor(log(5**power)/log(2)). + // The 128 comes from the ieee32 standard (the minimal exponent is -127). + // The 63 comes from the fact that we use a 64-bit word. + // See compute_float_64 for a discussion of the magical 152170 + 65536. + int64_t exponent = (((152170 + 65536) * power) >> 16) + 128 + 63; + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + // We want the most significant 64 bits of the product i * 5**power. It is + // safe to index the table because + // smallest_power <= smallest_power_binary32 <= power + // <= largest_power_binary32 <= largest_power. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index]); +#else + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); +#endif + + // Unless the least significant 38 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 26 bits are + // exact and no further work is needed. Having 26 bits is necessary because + // we need 24 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + // (For binary64, the same reasoning gives 55 bits and a 9-bit mask.) + if((firstproduct.high & 0x3FFFFFFFFF) == 0x3FFFFFFFFF) { + // The truncated multiplication was not accurate enough; use the next 64 + // bits of the power of five to refine it. See compute_float_64 for a + // detailed discussion. +#if SIMDJSON_STATIC_REFLECTION + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::powers_template<>::power_of_five_128[index + 1]); +#else + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); +#endif + firstproduct.low += secondproduct.high; + if(secondproduct.high > firstproduct.low) { firstproduct.high++; } + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 24 bits with a leading 1. + // We shift it so that it occupies 25 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 38); // 38 == 64 - 23 - 3 + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<25). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here we have that real_exponent <= 0 so -real_exponent >= 0 + if(-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0f : 0.0f; + return true; + } + // next line is safe because -real_exponent + 1 < 64 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // As in compute_float_64, rounding up may take us out of the subnormal + // range, so we can only decide after rounding. + real_exponent = (mantissa < (uint64_t(1) << 23)) ? 0 : 1; + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; + } + // We have to round to even. The "to even" part is only a problem when we are + // right in between two floats, which we guard against. The bounds on the + // power of ten are those of fast_float's + // min_exponent_round_to_even()/max_exponent_round_to_even() for binary32: + // when q >= 0, (2m+1) must be divisible by 5^q with 5^q <= 2^25, so q <= 10; + // when q < 0, we need 2^24 x 5^{-q} < 2^{64}, so q >= -17. + if (simdjson_unlikely((lower <= 1) && (power >= -17) && (power <= 10) && ((mantissa & 3) == 1))) { + if((mantissa << (upperbit + 38)) == upper) { + mantissa &= ~uint64_t(1); // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<24), unless there was an overflow + if (mantissa >= (uint64_t(1) << 24)) { + mantissa = (uint64_t(1) << 23); + real_exponent++; + } + mantissa &= ~(uint64_t(1) << 23); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 254)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_float(uint32_t(mantissa), uint32_t(real_exponent), negative); + return true; +} + +#if SIMDJSON_ENABLE_NAN_INF +// Parses a nan or infinity. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, double& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + double inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} + +// Parses a nan or infinity as a binary32 value. Returns true on success, false on failure. +simdjson_unused simdjson_inline bool compute_nan_inf(const uint8_t* src, bool negative, float& d) noexcept { + if (atomparsing::is_valid_inf_atom(src)) { + float inf = std::numeric_limits::infinity(); + d = negative ? -inf : inf; + return true; + } + + if (atomparsing::is_valid_nan_atom(src)) { + d = std::numeric_limits::quiet_NaN(); + return true; + } + + return false; +} +#endif + // We call a fallback floating-point parser that might be slow. Note // it will accept JSON numbers, but the JSON spec. is more restrictive so // before you call parse_float_fallback, you need to have validated the input @@ -37015,6 +43634,16 @@ static bool parse_float_fallback(const uint8_t *ptr, const uint8_t *end_ptr, dou return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); } +// Same as parse_float_fallback, but for binary32 (float) values. Going through +// the binary64 fallback and then rounding to binary32 would be subject to +// double rounding, so we run the fallback algorithm directly on binary32. +static bool parse_float_fallback(const uint8_t *ptr, float *outFloat) { + *outFloat = simdjson::internal::from_chars_float(reinterpret_cast(ptr)); + // We do not accept infinite values. See the binary64 version above for why we + // do not use std::isfinite. + return !(*outFloat > (std::numeric_limits::max)() || *outFloat < std::numeric_limits::lowest()); +} + // check quickly whether the next 8 chars are made of digits // at a glance, it looks better than Mula's // http://0x80.pl/articles/swar-digits-validate.html @@ -37033,6 +43662,28 @@ simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t *chars) { 0x3333333333333333); } +// The same idea for four characters. A block of eight only bites when eight +// digits are left, so a run of 4 to 7 digits used to fall back to one digit at a +// time; taking four of them at once is what makes the tail of a long fraction +// cheap. Adapted from fast_float, credit @aqrit. +simdjson_inline bool is_made_of_four_digits_fast(const uint8_t *chars) { + uint32_t val; + // Reads up to 3 bytes beyond the digits, which SIMDJSON_PADDING covers. + static_assert(3 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 3"); + std::memcpy(&val, chars, 4); + return !(((val + 0x46464646) | (val - 0x30303030)) & 0x80808080); +} + +// Only call this when is_made_of_four_digits_fast() says the four characters are +// digits. Little-endian: guarded by SIMDJSON_SWAR_NUMBER_PARSING. +simdjson_inline uint32_t parse_four_digits_unrolled(const uint8_t *chars) { + uint32_t val; + std::memcpy(&val, chars, 4); + val -= 0x30303030; + val = (val * 10) + (val >> 8); + return (((val & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; +} + template SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later simdjson_inline bool parse_digit(const uint8_t c, I &i) { @@ -37049,26 +43700,69 @@ simdjson_inline bool is_digit(const uint8_t c) { return static_cast(c - '0') <= 9; } -simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t *const first_after_period = p; +// Consumes a run of digits into i, eight at a time while eight are available, +// then four, then one at a time. Overflow is deliberate: the caller counts the +// digits and falls back when there are too many for a 64-bit mantissa. +// +// Long runs of digits are what a float-heavy document is made of: the fraction +// of a binary64 printed to full precision is 15 to 17 digits, and reading those +// one at a time is the single largest cost in parse_double(). Integer parts are +// left alone: they are usually a handful of digits, and there the check for a +// block of eight is wasted work. +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_fraction_digits(const uint8_t *&p, uint64_t &i) { +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + while (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } + // A 4 to 7 digit remainder is the common case once the blocks of eight are + // gone: 15 fraction digits leave 7, and 17 leave 1 after two blocks. + if (is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + while (parse_digit(*p, i)) { p++; } +} + +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_float_integer_digits(const uint8_t *&p, uint64_t &i, + bool &leading_zero) { + if (!parse_digit(*p, i)) { leading_zero = true; return; } + p++; + leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } +} +SIMDJSON_NO_SANITIZE_UNDEFINED +simdjson_inline void parse_integer_digits(const uint8_t *&p, uint64_t &i) { #ifdef SIMDJSON_SWAR_NUMBER_PARSING #if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. + // Identifiers, timestamps and counters often have eight digits or more. if (is_made_of_eight_digits_fast(p)) { i = i * 100000000 + parse_eight_digits_unrolled(p); p += 8; } + const uint8_t *const swar_end = p + 8; + while (p < swar_end && is_made_of_four_digits_fast(p)) { + i = i * 10000 + parse_four_digits_unrolled(p); + p += 4; + } #endif // SIMDJSON_SWAR_NUMBER_PARSING #endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } while (parse_digit(*p, i)) { p++; } +} + +simdjson_warn_unused simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t *const src, const uint8_t *&p, uint64_t &i, int64_t &exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t *const first_after_period = p; + parse_fraction_digits(p, i); exponent = first_after_period - p; // Decimal without digits (123.) is illegal if (exponent == 0) { @@ -37157,7 +43851,7 @@ simdjson_inline size_t significant_digits(const uint8_t * start_digits, size_t d } // unnamed namespace /** @private */ -static error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { +inline error_code slow_float_parsing(simdjson_unused const uint8_t * src, double* answer) { if (parse_float_fallback(src, answer)) { return SUCCESS; } @@ -37240,15 +43934,17 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons return SUCCESS; // always succeeds } -simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const src) noexcept { return 0; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { return false; } -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { return number_type::signed_integer; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * const) noexcept { return 0; } +simdjson_unused simdjson_inline bool is_negative(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t *) noexcept { return false; } +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t *) noexcept { return number_type::signed_integer; } #else // parse the number at src @@ -37279,7 +43975,21 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { + +#if SIMDJSON_ENABLE_NAN_INF + // By this point, we know that our input does not begin with a digit. We will attempt + // to handle NaN/Infinity. + + double d; + if (compute_nan_inf(p, negative, d)) { + writer.append_double(d); + return SUCCESS; + } +#endif + + return INVALID_NUMBER(src); + } // // Handle floats if there is a . or e (or both) @@ -37328,7 +44038,7 @@ simdjson_warn_unused simdjson_inline error_code parse_number(const uint8_t *cons // - Therefore, if the number is positive and lower than that, it's overflow. // - The value we are looking at is less than or equal to INT64_MAX. // - } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return BIGINT_NUMBER(src); } } // Write unsigned if it does not fit in a signed integer. @@ -37427,7 +44137,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned(const u // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37525,7 +44235,7 @@ simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_stri // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37580,7 +44290,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer(const uin // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = p; uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } + parse_integer_digits(p, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37666,7 +44376,7 @@ simdjson_unused simdjson_inline simdjson_result parse_integer_in_string // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare const uint8_t *const start_digits = src; uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } + parse_integer_digits(src, i); // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. // Optimization note: size_t is expected to be unsigned. @@ -37706,11 +44416,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } + if ( p == src ) { + +#if SIMDJSON_ENABLE_NAN_INF + // If there are no loading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + double d; + if (compute_nan_inf(p, negative, d)) { return d; } +#endif + + return INCORRECT_TYPE; + } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -37721,9 +44440,8 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -37772,67 +44490,12 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 return d; } -simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { - return (*src == '-'); -} - -simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; -} - -simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t *p = src; - while(static_cast(*p - '0') <= 9) { p++; } - size_t digit_count = size_t(p - src); - if ( p == src ) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); - // We have an integer. - if(simdjson_unlikely(digit_count > 20)) { - return number_type::big_integer; - } - // If the number is negative and valid, it must be a signed integer. - if(negative) { - if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; - if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { - return number_type::big_integer; - } -#if SIMDJSON_MINUS_ZERO_AS_FLOAT - if(digit_count == 1 && src[0] == '0') { - // We have to write -0.0 instead of 0 - return number_type::floating_point_number; - } -#endif - return number_type::signed_integer; - } - // Let us check if we have a big integer (>=2**64). - static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); - if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { - return number_type::big_integer; - } - // The number is positive and smaller than 18446744073709551616 (or 2**64). - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; -} - -// Never read at src_end or beyond -simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { - if(src == src_end) { return NUMBER_ERROR; } +// Parse a JSON number into a binary32 (float) value. +// +// This mirrors parse_double, but it rounds to binary32 directly instead of +// rounding to binary64 and then to binary32: the latter is subject to double +// rounding and would not always produce the nearest float. +simdjson_unused simdjson_inline simdjson_result parse_float(const uint8_t * src) noexcept { // // Check for minus sign // @@ -37844,91 +44507,20 @@ simdjson_unused simdjson_inline simdjson_result parse_double(const uint8 // uint64_t i = 0; const uint8_t *p = src; - if(p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } - if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + if ( p == src ) { - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t *start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p-src-1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t *start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits-src > 19; - } - } else { - overflow = p-src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if(p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t *start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0-exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, the number may be nan or infinity. + // Attempt to compute those, and return on success. + float f; + if (compute_nan_inf(p, negative, f)) { return f; } +#endif - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; + return INCORRECT_TYPE; } - return d; -} - -simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t *p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if ( p == src ) { return INCORRECT_TYPE; } if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } // @@ -37939,9 +44531,239 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c if (simdjson_likely(*p == '.')) { p++; const uint8_t *start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline bool is_negative(const uint8_t * src) noexcept { + return (*src == '-'); +} + +simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; +} + +simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t * src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t *p = src; + while(static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if ( p == src ) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t * smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if(simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if(negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } +#if SIMDJSON_MINUS_ZERO_AS_FLOAT + if(digit_count == 1 && src[0] == '0') { + // We have to write -0.0 instead of 0 + return number_type::floating_point_number; + } +#endif + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t * two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; +} + +// Never read at src_end or beyond +simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t * src, const uint8_t * const src_end) noexcept { + if(src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + if(p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { return INCORRECT_TYPE; } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t *start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits-src > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if(p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; +} + +simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + double inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits exponent = -(p - start_decimal_digits); // Overflow check. More than 19 digits (minus the decimal) may be overflow. @@ -37990,6 +44812,99 @@ simdjson_unused simdjson_inline simdjson_result parse_double_in_string(c return d; } +// Parse a JSON number held inside a JSON string into a binary32 (float) value. +// See parse_float for why we do not simply round parse_double_in_string. +simdjson_unused simdjson_inline simdjson_result parse_float_in_string(const uint8_t * src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t *p = src; + bool leading_zero; + parse_float_integer_digits(p, i, leading_zero); + // no integer digits, or 0123 (zero must be solo) + if ( p == src ) { +#if SIMDJSON_ENABLE_NAN_INF + // If there are no leading digits, attempt to parse numbers that are either + // NaN or Infinity + if (atomparsing::is_valid_inf_in_string(src)) { + float inf = std::numeric_limits::infinity(); + return negative ? -inf : inf; + } + + if (atomparsing::is_valid_nan_in_string(src)) { + return std::numeric_limits::quiet_NaN(); + } +#endif + + return INCORRECT_TYPE; + } + if ( (leading_zero && p != src+1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t *start_decimal_digits = p; + parse_fraction_digits(p, i); + if (p == start_decimal_digits) { return NUMBER_ERROR; } // no decimal digits + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p-src-1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t *start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p-start_digits > 19; + } + } else { + overflow = p-src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t *start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p-start_exp_digits == 0 || p-start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0-exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + float d; + if (simdjson_likely(!overflow)) { + if (compute_float_32(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; +} + } // unnamed namespace #endif // SIMDJSON_SKIPNUMBERPARSING @@ -38172,6 +45087,376 @@ simdjson_inline implementation_simdjson_result_base::implementation_simdjson_ // Otherwise, amalgamation will fail. /* skipped duplicate #include "simdjson/concepts.h" */ /* skipped duplicate #include "simdjson/dom/fractured_json.h" */ +/* including simdjson/annotations.h: #include "simdjson/annotations.h" */ +/* begin file simdjson/annotations.h */ +#ifndef SIMDJSON_ANNOTATIONS_H +#define SIMDJSON_ANNOTATIONS_H + +/** + * @file annotations.h + * @brief Provides compile-time annotations for simdjson structures. + * This header defines annotations that can be applied to data members of structures + * (and to the structures and enumerations themselves) to control how they are + * serialized/deserialized with simdjson. The set of annotations is modelled after + * the attributes of the Rust serde library. + * + * Member annotations: + * + * [[= simdjson::rename<"name">]] use "name" as the JSON key + * [[= simdjson::alias<"a", "b">]] also accept "a" and "b" when deserializing + * [[= simdjson::skip]] never serialize nor deserialize + * [[= simdjson::skip_serializing]] never serialize + * [[= simdjson::skip_deserializing]] never deserialize (keeps its current value) + * [[= simdjson::skip_serializing_if]] do not serialize when pred(value) is true + * [[= simdjson::default_value]] a missing key is not an error + * [[= simdjson::default_from]] a missing key sets the member to factory() + * [[= simdjson::with]] custom (de)serialization via Adapter + * [[= simdjson::flatten]] inline the members of a nested structure + * + * Structure (container) annotations: + * + * [[= simdjson::rename_all]] rename every member + * [[= simdjson::default_value]] no missing key is an error + * [[= simdjson::deny_unknown_fields]] unknown keys are a deserialization error + * [[= simdjson::transparent]] (de)serialize as the single member + * + * Enumeration annotations: rename_all on the enumeration, rename and alias on the + * enumerators (e.g., `enum class color { red [[= simdjson::rename<"RED">]] };`). + * + * This is currently experimental and subject to change (syntax and semantics may evolve). + */ + +#if SIMDJSON_STATIC_REFLECTION + +#include +#include +#include +#include + +namespace simdjson { + +// Structural compile-time string -- char array avoids the pointer-based +// 'reflect_constant failed' that occurs with const char* / string_view members. +template +struct fixed_string { + char data[N]; + + consteval fixed_string(const char (&s)[N]) noexcept { + for (size_t i = 0; i < N; ++i) { data[i] = s[i]; } + } + + consteval std::string_view view() const noexcept { return {data, N - 1}; } + + consteval bool operator==(const fixed_string&) const noexcept = default; +}; + +/** + * Naming conventions for simdjson::rename_all, mirroring serde's rename_all. + * Except for lowercase and uppercase, the C++ identifier is first split into + * words at underscores and at case changes ("userId", "user_id" and "UserId" all + * give the words "user" and "id"; "HTTPServer" gives "HTTP" and "Server"), and + * the words are then joined according to the convention. + */ +enum class case_style { + lowercase, ///< every letter lowercased, nothing else changes: userId -> userid + uppercase, ///< every letter uppercased, nothing else changes: user_id -> USER_ID + pascal_case, ///< UserId + camel_case, ///< userId + snake_case, ///< user_id + screaming_snake_case, ///< USER_ID + kebab_case, ///< user-id + screaming_kebab_case ///< USER-ID +}; + +namespace detail { + template + struct rename_t { + static constexpr auto name = Name; + // Exposed as a pointer and a size: std::meta::extract requires structural types. + static constexpr const char *key_data = Name.data; + static constexpr size_t key_size = Name.view().size(); + }; + template + struct alias_t { + static_assert(sizeof...(Names) > 0, "simdjson::alias requires at least one name"); + static constexpr std::string_view keys[] = {Names.view()...}; + static constexpr const std::string_view *keys_data = keys; + static constexpr size_t keys_count = sizeof...(Names); + }; + struct skip_tag {}; + struct skip_serializing_tag {}; + struct skip_deserializing_tag {}; + template + struct skip_serializing_if_t { + static constexpr auto predicate = Predicate; + }; + struct default_value_tag {}; + template + struct default_from_t { + static constexpr auto factory = Factory; + }; + template + struct with_t { + using adapter = Adapter; + }; + template + struct rename_all_t { + static constexpr case_style style = Style; + }; + struct deny_unknown_fields_tag {}; + struct transparent_tag {}; + struct flatten_tag {}; + + // Predicates usable with skip_serializing_if. + struct is_none_t { + template + constexpr bool operator()(const T& v) const noexcept { return !v; } + }; + struct is_empty_t { + template + constexpr bool operator()(const T& v) const noexcept { return v.empty(); } + }; +} // namespace detail + +// Usage: [[= simdjson::rename<"first_name">]] std::string firstName; +template +inline constexpr detail::rename_t rename{}; + +// Usage: [[= simdjson::alias<"userName", "login">]] std::string user_name; +// The aliases are accepted (in addition to the regular key) when deserializing. +// Serialization always uses the regular key. If the JSON object contains more +// than one of the names, which one is used is unspecified. +template +inline constexpr detail::alias_t alias{}; + +// Usage: [[= simdjson::skip]] int internalCache; +inline constexpr detail::skip_tag skip{}; + +// Usage: [[= simdjson::skip_serializing]] std::string password; +inline constexpr detail::skip_serializing_tag skip_serializing{}; + +// Usage: [[= simdjson::skip_deserializing]] int computed; +// The member is never assigned during deserialization (it keeps its current +// value) and a matching key in the JSON input is treated as unknown. +inline constexpr detail::skip_deserializing_tag skip_deserializing{}; + +// Usage: [[= simdjson::skip_serializing_if]] std::optional x; +// The predicate is called with the member value; when it returns true, the key +// is omitted from the output. +template +inline constexpr detail::skip_serializing_if_t skip_serializing_if{}; + +// Predicate: true for an empty std::optional, a null smart pointer, etc. +inline constexpr detail::is_none_t is_none{}; +// Predicate: true for an empty string or container. +inline constexpr detail::is_empty_t is_empty{}; + +// Usage: [[= simdjson::default_value]] int port = 8080; +// When the key is missing from the JSON input, the member is left untouched +// (with get(), it keeps its default member initializer) instead of reporting +// NO_SUCH_FIELD. Applied to a structure, it applies to all of its members. +inline constexpr detail::default_value_tag default_value{}; + +// Usage: [[= simdjson::default_from]] int port; +// When the key is missing from the JSON input, the member is assigned the +// result of calling the factory (a constexpr callable taking no argument, such +// as a captureless lambda or a pointer to a function). +template +inline constexpr detail::default_from_t default_from{}; + +// Usage: [[= simdjson::with]] std::chrono::system_clock::time_point t; +// Adapter is a type that provides one or both of +// static void serialize(simdjson::builder::string_builder &b, const T &value); +// static simdjson::error_code deserialize(simdjson::ondemand::value &v, T &out); +// (the parameters may also be declared auto&). When one of them is missing, the +// default behaviour is used in that direction. +template +inline constexpr detail::with_t with{}; + +// Usage: [[= simdjson::flatten]] pagination page; +// The members of the nested structure are (de)serialized as if they were members +// of the enclosing structure: {"id":1,"limit":10,"offset":0} rather than +// {"id":1,"page":{"limit":10,"offset":0}}. The nested structure's own annotations +// (rename_all, default_value, ...) apply to its members. +inline constexpr detail::flatten_tag flatten{}; + +// Usage: struct [[= simdjson::rename_all]] S {...}; +// Also applies to enumerations. An explicit rename on a member takes precedence. +template +inline constexpr detail::rename_all_t