cranelift: generalize inline copy to all constant-length bulk copies

Per review, move the inline expansion into `raw_bulk_memory_operation`, keyed
on a constant byte length, so it also covers `memory.copy` and `table.copy`,
not just `array.copy`. It now uses width-agnostic wide accesses (`i8x16` down
to `i8`), and `BulkOp::MemoryCopy` carries entity-appropriate flags. Threshold
128 bytes, from measurement (~1.7-2.7x faster up to 128 B, ties by 256).

Assisted-by: Claude Code:claude-opus-4-7

Author: gfx

crates/cranelift/src/func_environ.rs

@@ -3448,6 +3448,34 @@ impl FuncEnvironment<'_> {
     /// epochs are enabled to break up the copy into a loop of chunks with
     /// preemption checks between them.
     fn raw_bulk_memory_operation(&mut self, builder: &mut FunctionBuilder<'_>, mut op: BulkOp) {
+        // Fast path: a copy whose byte length is a small compile-time constant is
+        // expanded inline (see `emit_inline_memcpy`), skipping the libcall's fixed
+        // per-call cost (a wasm/host transition and an indirect call) that
+        // dominates tiny copies. Larger or dynamic copies, and all fills, use the
+        // libcall below, whose `memmove` amortizes that cost.
+        //
+        // The bound is empirical: measured on aarch64, inline is ~1.7-2.7x faster
+        // than the libcall through 128 bytes and ties by 256 (cost grows with the
+        // length, since every chunk is loaded before any is stored).
+        const INLINE_COPY_MAX_BYTES: u64 = 128;
+        if let BulkOp::MemoryCopy {
+            dst,
+            src,
+            len,
+            flags,
+        } = op
+        {
+            if let Some(bytes) = Self::value_as_const_int(builder, len) {
+                if (1..=INLINE_COPY_MAX_BYTES).contains(&bytes) {
+                    if self.tunables.consume_fuel {
+                        self.fuel_consumed += bytes as i64;
+                    }
+                    self.emit_inline_memcpy(builder, dst, src, bytes, flags);
+                    return;
+                }
+            }
+        }
+
         // Very scientifically chosen. Or, more seriously, this is just an
         // arbitrary number for now. 100k copies of this size locally takes half
         // a second, so seems like a reasonably large chunk size to not hit perf
@@ -3467,7 +3495,7 @@ impl FuncEnvironment<'_> {
                     env.epoch_check(builder);
                 }
                 match *op {
-                    BulkOp::MemoryCopy { dst, src, len } => {
+                    BulkOp::MemoryCopy { dst, src, len, .. } => {
                         if env.tunables.consume_fuel {
                             // Note that fuel is always a 64-bit counter.
                             let fuel_consumed = match env.pointer_type() {
@@ -3530,7 +3558,7 @@ impl FuncEnvironment<'_> {
             };
             let has_chunk = builder.ins().icmp(IntCC::UnsignedGreaterThan, len, chunk);
             match *op {
-                BulkOp::MemoryCopy { dst, src, len } => {
+                BulkOp::MemoryCopy { dst, src, len, .. } => {
                     builder.ins().brif(
                         has_chunk,
                         chunk_block,
@@ -3553,7 +3581,7 @@ impl FuncEnvironment<'_> {
         has_chunk_branch(builder, &op);
 
         let append_block_params = |builder: &mut FunctionBuilder<'_>, block, op: &mut _| match op {
-            BulkOp::MemoryCopy { dst, src, len } => {
+            BulkOp::MemoryCopy { dst, src, len, .. } => {
                 *dst = builder.append_block_param(block, pointer_type);
                 *src = builder.append_block_param(block, pointer_type);
                 *len = builder.append_block_param(block, pointer_type);
@@ -3577,7 +3605,7 @@ impl FuncEnvironment<'_> {
         *op_len = chunk;
         raw_call(self, builder, &op);
         match &mut op {
-            BulkOp::MemoryCopy { dst, src, len } => {
+            BulkOp::MemoryCopy { dst, src, len, .. } => {
                 *dst = builder.ins().iadd(*dst, chunk);
                 *src = builder.ins().iadd(*src, chunk);
                 *len = builder.ins().isub(remaining_len, chunk);
@@ -3983,12 +4011,18 @@ impl FuncEnvironment<'_> {
                     src_entity,
                     CheckedEntity::Memory(_) | CheckedEntity::Data { .. }
                 ));
+                // Linear-memory access flags (little-endian, heap alias region),
+                // as in `prepare_addr`.
+                let flags = ir::MemFlagsData::new()
+                    .with_endianness(ir::Endianness::Little)
+                    .with_alias_region(Some(ir::AliasRegion::Heap));
                 self.raw_bulk_memory_operation(
                     builder,
                     BulkOp::MemoryCopy {
                         dst: dst_raw_addr,
                         src: src_raw_addr,
                         len: len_ptr,
+                        flags,
                     },
                 );
                 Ok(())
@@ -4300,45 +4334,23 @@ impl FuncEnvironment<'_> {
         }
 
         // For memcpy, that's easy, just call the intrinsic with the right
-        // parameters.
+        // parameters. The inline path in `raw_bulk_memory_operation` uses these
+        // per-entity flags: GC arrays trap on heap corruption, tables use the
+        // table alias region.
         if !type_forbids_memcpy && dst_element_size == src_element_size {
-            // Expand small, statically-sized copies inline to skip the
-            // `memory_copy` libcall's fixed per-call cost (a wasm/host transition
-            // and indirect call), which dominates for tiny copies. Dynamic or
-            // larger copies keep the libcall, whose `memmove` amortizes it. Only
-            // arrays (in the GC heap) qualify; tables stay on the libcall.
-            const INLINE_ARRAY_COPY_MAX_ELEMS: u64 = 8;
-            if let CheckedEntity::Array { .. } = dst_entity {
-                let elem_ty = match dst_element_size {
-                    1 => Some(ir::types::I8),
-                    2 => Some(ir::types::I16),
-                    4 => Some(ir::types::I32),
-                    8 => Some(ir::types::I64),
-                    16 => Some(ir::types::I8X16),
-                    _ => None,
-                };
-                if let (Some(elem_ty), Some(n)) =
-                    (elem_ty, Self::value_as_const_int(builder, copy_len))
-                {
-                    if (1..=INLINE_ARRAY_COPY_MAX_ELEMS).contains(&n) {
-                        self.emit_inline_array_copy(
-                            builder,
-                            dst_elem_addr,
-                            src_elem_addr,
-                            elem_ty,
-                            dst_element_size,
-                            n,
-                        );
-                        return Ok(());
-                    }
+            let flags = match dst_entity {
+                CheckedEntity::Array { .. } => {
+                    ir::MemFlagsData::new().with_trap_code(Some(TRAP_GC_HEAP_CORRUPT))
                 }
-            }
+                _ => ir::MemFlagsData::new().with_alias_region(Some(ir::AliasRegion::Table)),
+            };
             self.raw_bulk_memory_operation(
                 builder,
                 BulkOp::MemoryCopy {
                     dst: dst_elem_addr,
                     src: src_elem_addr,
                     len: dst_copy_byte_len,
+                    flags,
                 },
             );
             return Ok(());
@@ -4415,57 +4427,70 @@ impl FuncEnvironment<'_> {
         Ok(())
     }
 
-    /// If `value` is a compile-time constant integer (possibly behind a widening
-    /// cast of one, as inserted for array indices), return its raw bits.
-    /// Out-of-range values are left for the caller to reject.
-    fn value_as_const_int(builder: &FunctionBuilder<'_>, mut value: ir::Value) -> Option<u64> {
-        loop {
-            let inst = builder.func.dfg.value_def(value).inst()?;
-            match builder.func.dfg.insts[inst] {
-                ir::InstructionData::UnaryImm {
-                    opcode: ir::Opcode::Iconst,
-                    imm,
-                } => return Some(imm.bits().cast_unsigned()),
-                ir::InstructionData::Unary {
-                    opcode: ir::Opcode::Uextend,
-                    arg,
-                } => value = arg,
-                _ => return None,
-            }
-        }
+    /// If `value` is a compile-time constant integer — possibly behind the
+    /// widening/narrowing casts and constant multiply that length computations
+    /// insert (a byte length is `element_count * element_size`) — return its raw
+    /// bits. Out-of-range values are left for the caller to reject.
+    fn value_as_const_int(builder: &FunctionBuilder<'_>, value: ir::Value) -> Option<u64> {
+        let inst = builder.func.dfg.value_def(value).inst()?;
+        Some(match builder.func.dfg.insts[inst] {
+            ir::InstructionData::UnaryImm {
+                opcode: ir::Opcode::Iconst,
+                imm,
+            } => imm.bits().cast_unsigned(),
+            ir::InstructionData::Unary {
+                opcode: ir::Opcode::Uextend | ir::Opcode::Ireduce,
+                arg,
+            } => Self::value_as_const_int(builder, arg)?,
+            ir::InstructionData::BinaryImm64 {
+                opcode: ir::Opcode::ImulImm,
+                arg,
+                imm,
+            } => Self::value_as_const_int(builder, arg)?.wrapping_mul(imm.bits().cast_unsigned()),
+            _ => return None,
+        })
     }
 
-    /// Expand a small, statically-sized `array.copy` into inline loads then
-    /// stores, avoiding the `memory_copy` libcall.
+    /// Expand a copy of `bytes` (a small compile-time constant) into inline loads
+    /// then stores, avoiding the `memory_copy` libcall.
     ///
-    /// The copy is bitwise: `elem_ty` is an integer or vector type matching the
-    /// element width (`f32`/`f64` use `i32`/`i64`, `v128` uses `i8x16`), so any
-    /// fixed-width element works. Every element is loaded before any is stored,
-    /// so overlapping ranges keep `array.copy`'s `memmove` semantics. The caller
-    /// has already bounds-checked the addresses and length.
-    fn emit_inline_array_copy(
+    /// The copy is bitwise and element-type agnostic: the byte range is covered
+    /// greedily with the widest convenient access (`i8x16` down to `i8`). Every
+    /// chunk is loaded before any is stored, so overlapping ranges keep `memmove`
+    /// semantics. `flags` must carry the entity-appropriate access flags and must
+    /// not assume alignment, since wide chunks may straddle element boundaries.
+    /// The caller has already bounds-checked the range.
+    fn emit_inline_memcpy(
         &mut self,
         builder: &mut FunctionBuilder<'_>,
         dst_addr: ir::Value,
         src_addr: ir::Value,
-        elem_ty: ir::Type,
-        elem_size: u32,
-        n: u64,
+        bytes: u64,
+        flags: ir::MemFlagsData,
     ) {
-        if self.tunables.consume_fuel {
-            self.fuel_consumed += n as i64;
-        }
-        // GC-heap access flags (trap on corruption, no alignment assumed). Not
-        // `GC_MEMFLAGS` directly: that constant is gated to the `gc` feature.
-        let flags = ir::MemFlagsData::new().with_trap_code(Some(TRAP_GC_HEAP_CORRUPT));
-        let stride = i32::try_from(elem_size).unwrap();
-        let count = i32::try_from(n).unwrap();
-        let mut vals: SmallVec<[ir::Value; 8]> = smallvec![];
-        for i in 0..count {
-            vals.push(builder.ins().load(elem_ty, flags, src_addr, i * stride));
+        const WIDTHS: &[(u64, ir::Type)] = &[
+            (16, ir::types::I8X16),
+            (8, ir::types::I64),
+            (4, ir::types::I32),
+            (2, ir::types::I16),
+            (1, ir::types::I8),
+        ];
+        let mut chunks: SmallVec<[(i32, ir::Type); 8]> = smallvec![];
+        let mut offset = 0u64;
+        let mut remaining = bytes;
+        for &(width, ty) in WIDTHS {
+            while remaining >= width {
+                chunks.push((i32::try_from(offset).unwrap(), ty));
+                offset += width;
+                remaining -= width;
+            }
         }
-        for (i, val) in (0..count).zip(vals) {
-            builder.ins().store(flags, val, dst_addr, i * stride);
+        let vals: SmallVec<[ir::Value; 8]> = chunks
+            .iter()
+            .map(|&(off, ty)| builder.ins().load(ty, flags, src_addr, off))
+            .collect();
+        for (&(off, _), val) in chunks.iter().zip(vals) {
+            builder.ins().store(flags, val, dst_addr, off);
         }
     }
 
@@ -5521,11 +5546,14 @@ enum BulkOp {
     /// A `memory.copy` operation, copying memory from `src` to `dst`.
     ///
     /// All of `dst`, `src`, and `len` must be pre-validated and inbounds. All
-    /// must have type `env.pointer_type()`.
+    /// must have type `env.pointer_type()`. `flags` are the access flags used if
+    /// the copy is expanded inline (see `emit_inline_memcpy`); they vary by the
+    /// entity being copied (GC heap, linear memory, or table).
     MemoryCopy {
         dst: ir::Value,
         src: ir::Value,
         len: ir::Value,
+        flags: ir::MemFlagsData,
     },
 
     /// A `memory.fill` operation, setting all bytes of `dst` to `val`.

tests/disas/array-copy-inline.wat

@@ -2,16 +2,17 @@
 ;;! test = 'optimize'
 ;;! flags = '-Wgc'
 
-;; A statically-sized small `array.copy` is expanded inline as a sequence of
-;; loads followed by stores (every element is loaded before any is stored so
-;; overlapping ranges still copy correctly), instead of calling the
-;; `memory_copy` libcall.
+;; A small, constant-length `array.copy` is expanded inline as wide loads
+;; followed by stores instead of calling the `memory_copy` libcall. The byte
+;; range is covered greedily with the widest convenient access, so 7 `i32`s (28
+;; bytes) become an `i8x16` + `i64` + `i32`, and every chunk is loaded before any
+;; is stored so overlapping ranges still copy correctly.
 
 (module
   (type $a (array (mut i32)))
 
   (func $copy (param (ref $a) i32 (ref $a) i32)
-    (array.copy $a $a (local.get 0) (local.get 1) (local.get 2) (local.get 3) (i32.const 4))
+    (array.copy $a $a (local.get 0) (local.get 1) (local.get 2) (local.get 3) (i32.const 7))
   )
 )
 ;; function u0:0(i64 vmctx, i64, i32, i32, i32, i32) tail {
@@ -26,16 +27,16 @@
 ;;
 ;;                                 block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32):
 ;; @002a                               trapz v2, user16
-;; @002a                               v84 = load.i64 notrap aligned readonly can_move v0+8
-;; @002a                               v8 = load.i64 notrap aligned readonly can_move v84+32
+;; @002a                               v83 = load.i64 notrap aligned readonly can_move v0+8
+;; @002a                               v8 = load.i64 notrap aligned readonly can_move v83+32
 ;; @002a                               v7 = uextend.i64 v2
 ;; @002a                               v9 = iadd v8, v7
 ;; @002a                               v10 = iconst.i64 16
 ;; @002a                               v11 = iadd v9, v10  ; v10 = 16
 ;; @002a                               v12 = load.i32 user2 readonly v11
 ;; @002a                               v14 = uextend.i64 v3
-;;                                     v86 = iconst.i64 4
-;; @002a                               v17 = iadd v14, v86  ; v86 = 4
+;;                                     v85 = iconst.i64 7
+;; @002a                               v17 = iadd v14, v85  ; v85 = 7
 ;; @002a                               v13 = uextend.i64 v12
 ;; @002a                               v18 = icmp ugt v17, v13
 ;; @002a                               trapnz v18, user17
@@ -45,34 +46,33 @@
 ;; @002a                               v30 = iadd v28, v10  ; v10 = 16
 ;; @002a                               v31 = load.i32 user2 readonly v30
 ;; @002a                               v33 = uextend.i64 v5
-;; @002a                               v36 = iadd v33, v86  ; v86 = 4
+;; @002a                               v36 = iadd v33, v85  ; v85 = 7
 ;; @002a                               v32 = uextend.i64 v31
 ;; @002a                               v37 = icmp ugt v36, v32
 ;; @002a                               trapnz v37, user17
-;; @002a                               v49 = load.i64 notrap aligned v84+40
-;;                                     v80 = iconst.i64 20
-;; @002a                               v22 = iadd v9, v80  ; v80 = 20
-;;                                     v89 = iconst.i64 2
-;;                                     v96 = ishl v14, v89  ; v89 = 2
-;; @002a                               v25 = iadd v22, v96
-;; @002a                               v51 = uadd_overflow_trap v25, v10, user2  ; v10 = 16
+;; @002a                               v49 = load.i64 notrap aligned v83+40
+;;                                     v79 = iconst.i64 20
+;; @002a                               v22 = iadd v9, v79  ; v79 = 20
+;;                                     v93 = iconst.i64 2
+;;                                     v94 = ishl v14, v93  ; v93 = 2
+;; @002a                               v25 = iadd v22, v94
+;;                                     v98 = iconst.i64 28
+;; @002a                               v51 = uadd_overflow_trap v25, v98, user2  ; v98 = 28
 ;; @002a                               v50 = iadd v8, v49
 ;; @002a                               v52 = icmp ugt v51, v50
 ;; @002a                               trapnz v52, user2
-;; @002a                               v41 = iadd v28, v80  ; v80 = 20
-;;                                     v98 = ishl v33, v89  ; v89 = 2
-;; @002a                               v44 = iadd v41, v98
-;; @002a                               v56 = uadd_overflow_trap v44, v10, user2  ; v10 = 16
+;; @002a                               v41 = iadd v28, v79  ; v79 = 20
+;;                                     v96 = ishl v33, v93  ; v93 = 2
+;; @002a                               v44 = iadd v41, v96
+;; @002a                               v56 = uadd_overflow_trap v44, v98, user2  ; v98 = 28
 ;; @002a                               v57 = icmp ugt v56, v50
 ;; @002a                               trapnz v57, user2
-;; @002a                               v58 = load.i32 user2 v44
-;; @002a                               v59 = load.i32 user2 v44+4
-;; @002a                               v60 = load.i32 user2 v44+8
-;; @002a                               v61 = load.i32 user2 v44+12
+;; @002a                               v58 = load.i8x16 user2 v44
+;; @002a                               v59 = load.i64 user2 v44+16
+;; @002a                               v60 = load.i32 user2 v44+24
 ;; @002a                               store user2 v58, v25
-;; @002a                               store user2 v59, v25+4
-;; @002a                               store user2 v60, v25+8
-;; @002a                               store user2 v61, v25+12
+;; @002a                               store user2 v59, v25+16
+;; @002a                               store user2 v60, v25+24
 ;; @002e                               jump block1
 ;;
 ;;                                 block1: