Merge branch 'develop' into optimize-oj_dump_cstr-sse4

Author: samyron

.github/workflows/CI.yml

@@ -16,7 +16,7 @@ jobs:
       matrix:
         # Due to https://github.com/actions/runner/issues/849, we have to use quotes for '3.0'
         ruby:
-          - head
+          # - head
           - '3.4'
           - '3.3'
           - '3.2'
@@ -55,9 +55,10 @@ jobs:
           - ruby: 'truffleruby'
             gemfile: no_rails
             os: ubuntu
-          - ruby: 'ucrt'
-            gemfile: no_rails
-            os: windows
+          # no longer supported or until min stack size issue can be resolved
+          # - ruby: 'ucrt'
+          #   gemfile: no_rails
+          #   os: windows
 
     env:
       BUNDLE_GEMFILE: gemfiles/${{ matrix.gemfile }}.gemfile
@@ -66,7 +67,7 @@ jobs:
     continue-on-error: ${{ matrix.ruby == 'head'}}
     name: Ruby ${{ matrix.ruby }} (${{ matrix.os }}${{ matrix.gemfile != 'no_rails' && ', ' || '' }}${{ matrix.gemfile != 'no_rails' && matrix.gemfile || '' }})
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}

.github/workflows/clang-format.yml

@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
 
     - name: Run clang-format style check
       uses: jidicula/clang-format-action@v4.15.0

CHANGELOG.md

@@ -1,5 +1,13 @@
 # CHANGELOG
 
+## 3.16.13 - 2025-12-05
+
+- Fixed rails encoding for Hash and Array subclasses.
+
+## 3.16.12 - 2025-10-29
+
+- Fixed dump realloc bug that occurred when using the compat mode dump options.
+
 ## 3.16.11 - 2025-05-29
 
 - Fixed range encoding with the :circular option

ext/oj/dump.c

@@ -201,7 +201,7 @@ void initialize_neon(void) {
 }
 #endif
 
-#ifdef OJ_USE_SSE4_2
+#ifdef HAVE_SIMD_SSE4_2
 
 static size_t (*hibit_friendly_size_simd)(const uint8_t *str, size_t len) = NULL;
 static __m128i hibit_friendly_chars_sse42[8];
@@ -995,7 +995,7 @@ neon_update(const char *str, uint8x16x4_t *cmap_neon, int neon_table_size, bool
     return result;
 }
 
-#elif defined(OJ_USE_SSE4_2)
+#elif defined(HAVE_SIMD_SSE4_2)
 typedef struct _sse42_match_result {
     __m128i actions;
     bool    needs_escape;
@@ -1100,11 +1100,11 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
     size_t size;
     char  *cmap;
 #ifdef HAVE_SIMD_NEON
-    uint8x16x4_t *cmap_neon = NULL;
-    int           neon_table_size;
+    uint8x16x4_t *cmap_neon       = NULL;
+    int           neon_table_size = 0;
 #elif defined(OJ_USE_SSE4_2)
-    __m128i *cmap_sse42 = NULL;
-    int      sse42_tab_size;
+    __m128i      *cmap_sse42 = NULL;
+    int           sse42_tab_size;
 #endif /* HAVE_SIMD_NEON */
     const char *orig                  = str;
     bool        has_hi                = false;
@@ -1173,7 +1173,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
 #ifdef HAVE_SIMD_NEON
         cmap_neon       = hibit_friendly_chars_neon;
         neon_table_size = 2;
-#elif defined(OJ_USE_SSE4_2)
+#elif defined(HAVE_SIMD_SSE4_2)
         cmap_sse42     = hibit_friendly_chars_sse42;
         sse42_tab_size = 8;
 #endif /* HAVE_NEON_SIMD */
@@ -1204,7 +1204,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
             *out->cur++ = ':';
         }
 
-#if defined(HAVE_SIMD_NEON) || defined(OJ_USE_SSE4_2)
+#if defined(HAVE_SIMD_NEON) || defined(HAVE_SIMD_SSE4_2)
 
 #define SEARCH_FLUSH                                  \
     if (str > cursor) {                               \
@@ -1216,11 +1216,11 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         const char *chunk_end;
         const char *cursor = str;
         char        matches[16];
-#endif /* HAVE_SIMD_NEON || OJ_USE_SSE4_2 */
+#endif /* HAVE_SIMD_NEON || HAVE_SIMD_SSE4_2 */
 
 #if defined(HAVE_SIMD_NEON)
         bool use_simd = (cmap_neon != NULL && cnt >= (sizeof(uint8x16_t))) ? true : false;
-#elif defined(OJ_USE_SSE4_2)
+#elif defined(HAVE_SIMD_SSE4_2)
         bool use_simd = (cmap_sse42 != NULL && cnt >= (sizeof(__m128i))) ? true : false;
 #endif
 
@@ -1290,7 +1290,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
         }
 #endif
 
-#ifdef OJ_USE_SSE4_2
+#ifdef HAVE_SIMD_SSE4_2
         if (use_simd) {
             while (str < end) {
                 const char *chunk_ptr = NULL;
@@ -1333,7 +1333,7 @@ void oj_dump_cstr(const char *str, size_t cnt, bool is_sym, bool escape1, Out ou
             }
             SEARCH_FLUSH;
         }
-#endif /* OJ_USE_SSE4_2 */
+#endif /* HAVE_SIMD_SSE4_2 */
 
         for (; str < end; str++) {
             str = process_character(cmap[(uint8_t)*str], str, end, out, orig, do_unicode_validation, &check_start);

ext/oj/dump_compat.c

@@ -148,10 +148,10 @@ static void dump_array(VALUE a, int depth, Out out, bool as_ok) {
         } else {
             size = d2 * out->indent + 2;
         }
-        assure_size(out, size * cnt);
         cnt--;
         for (i = 0; i <= cnt; i++) {
             if (out->opts->dump_opts.use) {
+                assure_size(out, size);
                 if (0 < out->opts->dump_opts.array_size) {
                     APPEND_CHARS(out->cur, out->opts->dump_opts.array_nl, out->opts->dump_opts.array_size);
                 }

ext/oj/extconf.rb

@@ -35,13 +35,12 @@
 
 dflags['OJ_DEBUG'] = true unless ENV['OJ_DEBUG'].nil?
 
-if with_config('--with-sse42')
-  if try_cflags('-msse4.2')
-    $CPPFLAGS += ' -msse4.2'
-    dflags['OJ_USE_SSE4_2'] = 1
-  else
-    warn 'SSE 4.2 is not supported on this platform.'
-  end
+# Enable SIMD optimizations - try SSE4.2 on x86_64 for best performance
+# Falls back to SSE2 or compiler defaults if not available
+if try_cflags('-msse4.2')
+  $CPPFLAGS += ' -msse4.2'
+elsif try_cflags('-msse2')
+  $CPPFLAGS += ' -msse2'
 end
 
 if enable_config('trace-log', false)

ext/oj/parse.c

@@ -15,12 +15,9 @@
 #include "mem.h"
 #include "oj.h"
 #include "rxclass.h"
+#include "simd.h"
 #include "val_stack.h"
 
-#ifdef OJ_USE_SSE4_2
-#include <nmmintrin.h>
-#endif
-
 // Workaround in case INFINITY is not defined in math.h or if the OS is CentOS
 #define OJ_INFINITY (1.0 / 0.0)
 
@@ -202,23 +199,143 @@ static inline const char *scan_string_noSIMD(const char *str, const char *end) {
     return str;
 }
 
-#ifdef OJ_USE_SSE4_2
-static inline const char *scan_string_SIMD(const char *str, const char *end) {
-    static const char chars[16] = "\x00\\\"";
-    const __m128i     terminate = _mm_loadu_si128((const __m128i *)&chars[0]);
-    const char       *_end      = (const char *)(end - 16);
+#ifdef HAVE_SIMD_SSE4_2
+// Optimized SIMD string scanner using SSE4.2 instructions
+// Uses prefetching and processes multiple chunks in parallel to reduce latency
+static inline const char *scan_string_SSE42(const char *str, const char *end) {
+    static const char chars[16]   = "\x00\\\"";
+    const __m128i     terminate   = _mm_loadu_si128((const __m128i *)&chars[0]);
+    const char       *safe_end_64 = end - 64;
+    const char       *safe_end_16 = end - 16;
+
+    // Process 64 bytes at a time with parallel SIMD operations
+    // This reduces pipeline stalls and improves instruction-level parallelism
+    while (str <= safe_end_64) {
+        // Prefetch next cache line for better memory throughput
+        __builtin_prefetch(str + 64, 0, 0);
+
+        // Load and compare 4 chunks in parallel
+        const __m128i chunk0 = _mm_loadu_si128((const __m128i *)(str));
+        const __m128i chunk1 = _mm_loadu_si128((const __m128i *)(str + 16));
+        const __m128i chunk2 = _mm_loadu_si128((const __m128i *)(str + 32));
+        const __m128i chunk3 = _mm_loadu_si128((const __m128i *)(str + 48));
+
+        const int r0 = _mm_cmpestri(terminate,
+                                    3,
+                                    chunk0,
+                                    16,
+                                    _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (__builtin_expect(r0 != 16, 0))
+            return str + r0;
+
+        const int r1 = _mm_cmpestri(terminate,
+                                    3,
+                                    chunk1,
+                                    16,
+                                    _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (__builtin_expect(r1 != 16, 0))
+            return str + 16 + r1;
+
+        const int r2 = _mm_cmpestri(terminate,
+                                    3,
+                                    chunk2,
+                                    16,
+                                    _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (__builtin_expect(r2 != 16, 0))
+            return str + 32 + r2;
+
+        const int r3 = _mm_cmpestri(terminate,
+                                    3,
+                                    chunk3,
+                                    16,
+                                    _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
+        if (__builtin_expect(r3 != 16, 0))
+            return str + 48 + r3;
+
+        str += 64;
+    }
 
-    for (; str <= _end; str += 16) {
+    // Handle remaining 16-byte chunks
+    for (; str <= safe_end_16; str += 16) {
         const __m128i string = _mm_loadu_si128((const __m128i *)str);
         const int     r      = _mm_cmpestri(terminate,
                                    3,
                                    string,
                                    16,
                                    _SIDD_UBYTE_OPS | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT);
-        if (r != 16) {
-            str = (char *)(str + r);
-            return str;
-        }
+        if (r != 16)
+            return str + r;
+    }
+
+    return scan_string_noSIMD(str, end);
+}
+#endif
+
+#ifdef HAVE_SIMD_SSE2
+// Optimized SSE2 string scanner (fallback for older x86_64 CPUs)
+// Uses SSE2 instructions with prefetching and parallel processing
+static inline const char *scan_string_SSE2(const char *str, const char *end) {
+    const char *safe_end_64 = end - 64;
+    const char *safe_end_16 = end - 16;
+
+    // Create comparison vectors for our three special characters
+    const __m128i null_char = _mm_setzero_si128();
+    const __m128i backslash = _mm_set1_epi8('\\');
+    const __m128i quote     = _mm_set1_epi8('"');
+
+    // Process 64 bytes at a time for better throughput
+    while (str <= safe_end_64) {
+        __builtin_prefetch(str + 64, 0, 0);
+
+        // Load 4 chunks
+        const __m128i chunk0 = _mm_loadu_si128((const __m128i *)(str));
+        const __m128i chunk1 = _mm_loadu_si128((const __m128i *)(str + 16));
+        const __m128i chunk2 = _mm_loadu_si128((const __m128i *)(str + 32));
+        const __m128i chunk3 = _mm_loadu_si128((const __m128i *)(str + 48));
+
+        // Compare all chunks (allows CPU to parallelize)
+        const __m128i cmp0 = _mm_or_si128(
+            _mm_or_si128(_mm_cmpeq_epi8(chunk0, null_char), _mm_cmpeq_epi8(chunk0, backslash)),
+            _mm_cmpeq_epi8(chunk0, quote));
+        const __m128i cmp1 = _mm_or_si128(
+            _mm_or_si128(_mm_cmpeq_epi8(chunk1, null_char), _mm_cmpeq_epi8(chunk1, backslash)),
+            _mm_cmpeq_epi8(chunk1, quote));
+        const __m128i cmp2 = _mm_or_si128(
+            _mm_or_si128(_mm_cmpeq_epi8(chunk2, null_char), _mm_cmpeq_epi8(chunk2, backslash)),
+            _mm_cmpeq_epi8(chunk2, quote));
+        const __m128i cmp3 = _mm_or_si128(
+            _mm_or_si128(_mm_cmpeq_epi8(chunk3, null_char), _mm_cmpeq_epi8(chunk3, backslash)),
+            _mm_cmpeq_epi8(chunk3, quote));
+
+        // Convert to masks
+        int mask0 = _mm_movemask_epi8(cmp0);
+        if (__builtin_expect(mask0 != 0, 0))
+            return str + __builtin_ctz(mask0);
+
+        int mask1 = _mm_movemask_epi8(cmp1);
+        if (__builtin_expect(mask1 != 0, 0))
+            return str + 16 + __builtin_ctz(mask1);
+
+        int mask2 = _mm_movemask_epi8(cmp2);
+        if (__builtin_expect(mask2 != 0, 0))
+            return str + 32 + __builtin_ctz(mask2);
+
+        int mask3 = _mm_movemask_epi8(cmp3);
+        if (__builtin_expect(mask3 != 0, 0))
+            return str + 48 + __builtin_ctz(mask3);
+
+        str += 64;
+    }
+
+    // Handle remaining 16-byte chunks
+    for (; str <= safe_end_16; str += 16) {
+        const __m128i chunk   = _mm_loadu_si128((const __m128i *)str);
+        const __m128i matches = _mm_or_si128(
+            _mm_or_si128(_mm_cmpeq_epi8(chunk, null_char), _mm_cmpeq_epi8(chunk, backslash)),
+            _mm_cmpeq_epi8(chunk, quote));
+        int mask = _mm_movemask_epi8(matches);
+        if (mask != 0)
+            return str + __builtin_ctz(mask);
     }
 
     return scan_string_noSIMD(str, end);
@@ -228,9 +345,12 @@ static inline const char *scan_string_SIMD(const char *str, const char *end) {
 static const char *(*scan_func)(const char *str, const char *end) = scan_string_noSIMD;
 
 void oj_scanner_init(void) {
-#ifdef OJ_USE_SSE4_2
-    scan_func = scan_string_SIMD;
+#ifdef HAVE_SIMD_SSE4_2
+    scan_func = scan_string_SSE42;
+#elif defined(HAVE_SIMD_SSE2)
+    scan_func = scan_string_SSE2;
 #endif
+    // Note: ARM NEON string scanning would be added here if needed
 }
 
 // entered at /

ext/oj/rails.c

@@ -661,13 +661,15 @@ static VALUE encoder_new(int argc, VALUE *argv, VALUE self) {
     Encoder e = OJ_R_ALLOC(struct _encoder);
 
     e->opts = oj_default_options;
-    e->arg  = Qnil;
     copy_opts(&ropts, &e->ropts);
 
     if (1 <= argc && Qnil != *argv) {
-        oj_parse_options(*argv, &e->opts);
         e->arg = *argv;
+    } else {
+        e->arg = rb_hash_new();
     }
+    oj_parse_options(e->arg, &e->opts);
+
     return TypedData_Wrap_Struct(encoder_class, &oj_encoder_type, e);
 }