[JSC] Use simde in JSON

https://bugs.webkit.org/show_bug.cgi?id=273838
rdar://127688057

Reviewed by Keith Miller.

This patch implements following two things via simde.

1. Fast scanning of strings in JSON.parse.
2. Fast scanning and copying strings in JSON.stringify.

* Source/JavaScriptCore/runtime/JSONObject.cpp:
(JSC::FastStringifier<CharType>::append):
* Source/JavaScriptCore/runtime/LiteralParser.cpp:
(JSC::LiteralParser<CharType>::Lexer::lexString):
* Source/WTF/wtf/text/StringCommon.h:

Canonical link: https://commits.webkit.org/278494@main

Author: Constellation

Source/JavaScriptCore/runtime/JSONObject.cpp

@@ -41,6 +41,7 @@
 #include <charconv>
 #include <wtf/text/EscapedFormsForJSON.h>
 #include <wtf/text/StringBuilder.h>
+#include <wtf/text/StringCommon.h>
 
 // Turn this on to log information about fastStringify usage, with a focus on why it failed.
 #define FAST_STRINGIFY_LOG_USAGE 0
@@ -1059,6 +1060,102 @@ void FastStringifier<CharType>::append(JSValue value)
             recordFailure("String::tryGetValue"_s);
             return;
         }
+
+        auto charactersCopySameType = [&](auto span, auto* cursor) ALWAYS_INLINE_LAMBDA {
+#if CPU(ARM64) || CPU(X86_64)
+            constexpr size_t stride = 16 / sizeof(CharType);
+            if (span.size() >= stride) {
+                using UnsignedType = std::make_unsigned_t<CharType>;
+                using BulkType = decltype(WTF::loadBulk(static_cast<const UnsignedType*>(nullptr)));
+                constexpr auto quoteMask = WTF::splatBulk(static_cast<UnsignedType>('"'));
+                constexpr auto escapeMask = WTF::splatBulk(static_cast<UnsignedType>('\\'));
+                constexpr auto controlMask = WTF::splatBulk(static_cast<UnsignedType>(' '));
+                const auto* ptr = span.data();
+                const auto* end = ptr + span.size();
+                auto* cursorEnd = cursor + span.size();
+                BulkType accumulated { };
+                for (; ptr + (stride - 1) < end; ptr += stride, cursor += stride) {
+                    auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(ptr));
+                    WTF::storeBulk(input, bitwise_cast<UnsignedType*>(cursor));
+                    auto quotes = WTF::equalBulk(input, quoteMask);
+                    auto escapes = WTF::equalBulk(input, escapeMask);
+                    auto controls = WTF::lessThanBulk(input, controlMask);
+                    accumulated = WTF::mergeBulk(accumulated, WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls)));
+                    if constexpr (sizeof(CharType) != 1) {
+                        constexpr auto surrogateMask = WTF::splatBulk(static_cast<UnsignedType>(0xf800));
+                        constexpr auto surrogateCheckMask = WTF::splatBulk(static_cast<UnsignedType>(0xd800));
+                        accumulated = WTF::mergeBulk(accumulated, WTF::equalBulk(simde_vandq_u16(input, surrogateMask), surrogateCheckMask));
+                    }
+                }
+                if (ptr < end) {
+                    auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(end - stride));
+                    WTF::storeBulk(input, bitwise_cast<UnsignedType*>(cursorEnd - stride));
+                    auto quotes = WTF::equalBulk(input, quoteMask);
+                    auto escapes = WTF::equalBulk(input, escapeMask);
+                    auto controls = WTF::lessThanBulk(input, controlMask);
+                    accumulated = WTF::mergeBulk(accumulated, WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls)));
+                    if constexpr (sizeof(CharType) != 1) {
+                        constexpr auto surrogateMask = WTF::splatBulk(static_cast<UnsignedType>(0xf800));
+                        constexpr auto surrogateCheckMask = WTF::splatBulk(static_cast<UnsignedType>(0xd800));
+                        accumulated = WTF::mergeBulk(accumulated, WTF::equalBulk(simde_vandq_u16(input, surrogateMask), surrogateCheckMask));
+                    }
+                }
+                return WTF::isNonZeroBulk(accumulated);
+            }
+#endif
+            for (auto character : span) {
+                if constexpr (sizeof(CharType) != 1) {
+                    if (UNLIKELY(U16_IS_SURROGATE(character)))
+                        return true;
+                }
+                if (UNLIKELY(character <= 0xff && WTF::escapedFormsForJSON[character]))
+                    return true;
+                *cursor++ = character;
+            }
+            return false;
+        };
+
+        auto charactersCopyUpconvert = [&](std::span<const LChar> span, UChar* cursor) ALWAYS_INLINE_LAMBDA {
+#if CPU(ARM64) || CPU(X86_64)
+            constexpr size_t stride = 16 / sizeof(LChar);
+            if (span.size() >= stride) {
+                using UnsignedType = std::make_unsigned_t<LChar>;
+                using BulkType = decltype(WTF::loadBulk(static_cast<const UnsignedType*>(nullptr)));
+                constexpr auto quoteMask = WTF::splatBulk(static_cast<UnsignedType>('"'));
+                constexpr auto escapeMask = WTF::splatBulk(static_cast<UnsignedType>('\\'));
+                constexpr auto controlMask = WTF::splatBulk(static_cast<UnsignedType>(' '));
+                constexpr auto zeros = WTF::splatBulk(static_cast<UnsignedType>(0));
+                const auto* ptr = span.data();
+                const auto* end = ptr + span.size();
+                auto* cursorEnd = cursor + span.size();
+                BulkType accumulated { };
+                for (; ptr + (stride - 1) < end; ptr += stride, cursor += stride) {
+                    auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(ptr));
+                    simde_vst2q_u8(bitwise_cast<UnsignedType*>(cursor), (simde_uint8x16x2_t { input, zeros }));
+                    auto quotes = WTF::equalBulk(input, quoteMask);
+                    auto escapes = WTF::equalBulk(input, escapeMask);
+                    auto controls = WTF::lessThanBulk(input, controlMask);
+                    accumulated = WTF::mergeBulk(accumulated, WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls)));
+                }
+                if (ptr < end) {
+                    auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(end - stride));
+                    simde_vst2q_u8(bitwise_cast<UnsignedType*>(cursorEnd - stride), (simde_uint8x16x2_t { input, zeros }));
+                    auto quotes = WTF::equalBulk(input, quoteMask);
+                    auto escapes = WTF::equalBulk(input, escapeMask);
+                    auto controls = WTF::lessThanBulk(input, controlMask);
+                    accumulated = WTF::mergeBulk(accumulated, WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls)));
+                }
+                return WTF::isNonZeroBulk(accumulated);
+            }
+#endif
+            for (auto character : span) {
+                if (UNLIKELY(WTF::escapedFormsForJSON[character]))
+                    return true;
+                *cursor++ = character;
+            }
+            return false;
+        };
+
         if constexpr (sizeof(CharType) == 1) {
             if (UNLIKELY(!string.is8Bit())) {
                 m_retryWith16BitFastStringifier = m_length < (m_capacity / 2);
@@ -1070,47 +1167,32 @@ void FastStringifier<CharType>::append(JSValue value)
                 recordBufferFull();
                 return;
             }
-            auto* cursor = m_buffer + m_length;
-            *cursor++ = '"';
-            for (auto character : string.span8()) {
-                if (UNLIKELY(WTF::escapedFormsForJSON[character])) {
-                    recordFailure("string character needs escaping"_s);
-                    return;
-                }
-                *cursor++ = character;
+            m_buffer[m_length] = '"';
+            if (UNLIKELY(charactersCopySameType(string.span8(), m_buffer + m_length + 1))) {
+                recordFailure("string character needs escaping"_s);
+                return;
             }
-            *cursor = '"';
+            m_buffer[m_length + 1 + stringLength] = '"';
             m_length += 1 + stringLength + 1;
         } else {
             auto stringLength = string.length();
             if (UNLIKELY(!hasRemainingCapacity(1 + stringLength + 1))) {
                 recordBufferFull();
                 return;
             }
-            auto* cursor = m_buffer + m_length;
-            *cursor++ = '"';
+            m_buffer[m_length] = '"';
             if (string.is8Bit()) {
-                for (auto character : string.span8()) {
-                    if (UNLIKELY(WTF::escapedFormsForJSON[character])) {
-                        recordFailure("string character needs escaping"_s);
-                        return;
-                    }
-                    *cursor++ = character;
+                if (UNLIKELY(charactersCopyUpconvert(string.span8(), m_buffer + m_length + 1))) {
+                    recordFailure("string character needs escaping"_s);
+                    return;
                 }
             } else {
-                for (auto character : string.span16()) {
-                    if (UNLIKELY(U16_IS_SURROGATE(character))) {
-                        recordFailure("string character is surrogate"_s);
-                        return;
-                    }
-                    if (UNLIKELY(character <= 0xff && WTF::escapedFormsForJSON[character])) {
-                        recordFailure("string character needs escaping"_s);
-                        return;
-                    }
-                    *cursor++ = character;
+                if (UNLIKELY(charactersCopySameType(string.span16(), m_buffer + m_length + 1))) {
+                    recordFailure("string character needs escaping or surrogate pair handling"_s);
+                    return;
                 }
             }
-            *cursor = '"';
+            m_buffer[m_length + 1 + stringLength] = '"';
             m_length += 1 + stringLength + 1;
         }
         return;

Source/JavaScriptCore/runtime/LiteralParser.cpp

@@ -867,8 +867,43 @@ ALWAYS_INLINE TokenType LiteralParser<CharType>::Lexer::lexString(LiteralParserT
             while (m_ptr < m_end && isSafeStringCharacterForIdentifier<SafeStringCharacterSet::Strict>(*m_ptr, '"'))
                 ++m_ptr;
         } else {
-            while (m_ptr < m_end && isSafeStringCharacter<SafeStringCharacterSet::Strict>(*m_ptr, '"'))
-                ++m_ptr;
+            ([&]() ALWAYS_INLINE_LAMBDA {
+#if CPU(ARM64) || CPU(X86_64)
+                constexpr size_t stride = 16 / sizeof(CharType);
+                using UnsignedType = std::make_unsigned_t<CharType>;
+                if (static_cast<size_t>(m_end - m_ptr) >= stride) {
+                    constexpr auto quoteMask = WTF::splatBulk(static_cast<UnsignedType>('"'));
+                    constexpr auto escapeMask = WTF::splatBulk(static_cast<UnsignedType>('\\'));
+                    constexpr auto controlMask = WTF::splatBulk(static_cast<UnsignedType>(' '));
+                    for (; m_ptr + (stride - 1) < m_end; m_ptr += stride) {
+                        auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(m_ptr));
+                        auto quotes = WTF::equalBulk(input, quoteMask);
+                        auto escapes = WTF::equalBulk(input, escapeMask);
+                        auto controls = WTF::lessThanBulk(input, controlMask);
+                        auto mask = WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls));
+                        if (WTF::isNonZeroBulk(mask)) {
+                            m_ptr += WTF::findFirstNonZeroIndexBulk(mask);
+                            return;
+                        }
+                    }
+                    if (m_ptr < m_end) {
+                        auto input = WTF::loadBulk(bitwise_cast<const UnsignedType*>(m_end - stride));
+                        auto quotes = WTF::equalBulk(input, quoteMask);
+                        auto escapes = WTF::equalBulk(input, escapeMask);
+                        auto controls = WTF::lessThanBulk(input, controlMask);
+                        auto mask = WTF::mergeBulk(quotes, WTF::mergeBulk(escapes, controls));
+                        if (WTF::isNonZeroBulk(mask)) {
+                            m_ptr = m_end - stride + WTF::findFirstNonZeroIndexBulk(mask);
+                            return;
+                        }
+                        m_ptr = m_end;
+                    }
+                    return;
+                }
+#endif
+                while (m_ptr < m_end && isSafeStringCharacter<SafeStringCharacterSet::Strict>(*m_ptr, '"'))
+                    ++m_ptr;
+            }());
         }
     } else {
         while (m_ptr < m_end && isSafeStringCharacter<SafeStringCharacterSet::Sloppy>(*m_ptr, terminator))

Source/WTF/wtf/text/StringCommon.h

@@ -1210,7 +1210,17 @@ inline void copyElements(LChar* __restrict destination, const UChar* __restrict
     copyElements(bitwise_cast<uint8_t*>(destination), bitwise_cast<const uint16_t*>(source), length);
 }
 
-#if CPU(ARM64)
+#if CPU(ARM64) || CPU(X86_64)
+
+constexpr simde_uint8x16_t splatBulk(uint8_t code)
+{
+    return simde_uint8x16_t { code, code, code, code, code, code, code, code, code, code, code, code, code, code, code, code };
+}
+
+constexpr simde_uint16x8_t splatBulk(uint16_t code)
+{
+    return simde_uint16x8_t { code, code, code, code, code, code, code, code };
+}
 
 ALWAYS_INLINE simde_uint8x16_t loadBulk(const uint8_t* ptr)
 {
@@ -1222,6 +1232,16 @@ ALWAYS_INLINE simde_uint16x8_t loadBulk(const uint16_t* ptr)
     return simde_vld1q_u16(ptr);
 }
 
+ALWAYS_INLINE void storeBulk(simde_uint8x16_t value, uint8_t* ptr)
+{
+    return simde_vst1q_u8(ptr, value);
+}
+
+ALWAYS_INLINE void storeBulk(simde_uint16x8_t value, uint16_t* ptr)
+{
+    return simde_vst1q_u16(ptr, value);
+}
+
 ALWAYS_INLINE simde_uint8x16_t mergeBulk(simde_uint8x16_t accumulated, simde_uint8x16_t input)
 {
     return simde_vorrq_u8(accumulated, input);
@@ -1242,24 +1262,56 @@ ALWAYS_INLINE bool isNonZeroBulk(simde_uint16x8_t accumulated)
     return simde_vmaxvq_u16(accumulated);
 }
 
+ALWAYS_INLINE uint8_t findFirstNonZeroIndexBulk(simde_uint8x16_t value)
+{
+    constexpr simde_uint8x16_t indexMask { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
+    return simde_vminvq_u8(simde_vornq_u8(indexMask, value));
+}
+
+ALWAYS_INLINE uint8_t findFirstNonZeroIndexBulk(simde_uint16x8_t value)
+{
+    constexpr simde_uint16x8_t indexMask { 0, 1, 2, 3, 4, 5, 6, 7 };
+    return simde_vminvq_u16(simde_vornq_u16(indexMask, value));
+}
+
 template<LChar character, LChar... characters>
-ALWAYS_INLINE simde_uint8x16_t compareBulk(simde_uint8x16_t input)
+ALWAYS_INLINE simde_uint8x16_t equalBulk(simde_uint8x16_t input)
 {
     auto result = simde_vceqq_u8(input, simde_vmovq_n_u8(character));
     if constexpr (!sizeof...(characters))
         return result;
     else
-        return mergeBulk(result, compareBulk<characters...>(input));
+        return mergeBulk(result, equalBulk<characters...>(input));
 }
 
 template<UChar character, UChar... characters>
-ALWAYS_INLINE simde_uint16x8_t compareBulk(simde_uint16x8_t input)
+ALWAYS_INLINE simde_uint16x8_t equalBulk(simde_uint16x8_t input)
 {
     auto result = simde_vceqq_u16(input, simde_vmovq_n_u16(character));
     if constexpr (!sizeof...(characters))
         return result;
     else
-        return mergeBulk(result, compareBulk<characters...>(input));
+        return mergeBulk(result, equalBulk<characters...>(input));
+}
+
+ALWAYS_INLINE simde_uint8x16_t equalBulk(simde_uint8x16_t lhs, simde_uint8x16_t rhs)
+{
+    return simde_vceqq_u8(lhs, rhs);
+}
+
+ALWAYS_INLINE simde_uint16x8_t equalBulk(simde_uint16x8_t lhs, simde_uint16x8_t rhs)
+{
+    return simde_vceqq_u16(lhs, rhs);
+}
+
+ALWAYS_INLINE simde_uint8x16_t lessThanBulk(simde_uint8x16_t lhs, simde_uint8x16_t rhs)
+{
+    return simde_vcltq_u8(lhs, rhs);
+}
+
+ALWAYS_INLINE simde_uint16x8_t lessThanBulk(simde_uint16x8_t lhs, simde_uint16x8_t rhs)
+{
+    return simde_vcltq_u16(lhs, rhs);
 }
 
 #endif
@@ -1277,18 +1329,18 @@ ALWAYS_INLINE bool charactersContain(std::span<const CharacterType> span)
     auto* data = span.data();
     size_t length = span.size();
 
-#if CPU(ARM64)
+#if CPU(ARM64) || CPU(X86_64)
     constexpr size_t stride = 16 / sizeof(CharacterType);
     using UnsignedType = std::make_unsigned_t<CharacterType>;
     using BulkType = decltype(loadBulk(static_cast<const UnsignedType*>(nullptr)));
     if (length >= stride) {
         size_t index = 0;
         BulkType accumulated { };
         for (; index + (stride - 1) < length; index += stride)
-            accumulated = mergeBulk(accumulated, compareBulk<characters...>(loadBulk(bitwise_cast<const UnsignedType*>(data + index))));
+            accumulated = mergeBulk(accumulated, equalBulk<characters...>(loadBulk(bitwise_cast<const UnsignedType*>(data + index))));
 
         if (index < length)
-            accumulated = mergeBulk(accumulated, compareBulk<characters...>(loadBulk(bitwise_cast<const UnsignedType*>(data + length - stride))));
+            accumulated = mergeBulk(accumulated, equalBulk<characters...>(loadBulk(bitwise_cast<const UnsignedType*>(data + length - stride))));
 
         return isNonZeroBulk(accumulated);
     }