Merge remote-tracking branch 'origin/main' into gfx/branch-hinting-v2

# Conflicts:
#	crates/cranelift/src/compiler.rs
#	crates/cranelift/src/func_environ.rs
#	crates/environ/src/compile/module_environ.rs

Author: gfx

.github/workflows/main.yml

@@ -423,6 +423,8 @@ jobs:
               -p wasmtime-wasi-http --no-default-features
               -p wasmtime-wasi-http --no-default-features --features p2
               -p wasmtime-wasi-http --no-default-features --features p3
+              -p wasmtime-wasi-http --no-default-features --features p2,component-model-async
+              -p wasmtime-wasi-http --no-default-features --features p3,component-model-async
               -p wasmtime-wasi-http --no-default-features --features p3 --all-targets
 
           - name: wasmtime-wasi
@@ -605,7 +607,7 @@ jobs:
             cargo check -p wasmtime --no-default-features --features runtime,gc,component-model,pulley,async,debug,debug-builtins,demangle,anyhow &&
             cargo check -p cranelift-control --no-default-features &&
             cargo check -p cranelift-assembler-x64 --lib &&
-            cargo check -p cranelift-codegen --no-default-features --features x86,arm64 &&
+            cargo check -p cranelift-codegen --no-default-features --features x86,arm64,riscv64 &&
             cargo check -p cranelift-frontend --no-default-features &&
             cargo check -p pulley-interpreter --features encode,decode,disas,interp &&
             cargo check -p wasmtime-wasi-io --no-default-features

Cargo.lock

@@ -91,6 +91,7 @@ hyper = { workspace = true, optional = true }
 http = { workspace = true, optional = true }
 http-body-util = { workspace = true, optional = true }
 futures = { workspace = true, optional = true }
+pin-project-lite = { workspace = true, optional = true }
 
 [target.'cfg(unix)'.dependencies]
 rustix = { workspace = true, features = ["mm", "process"] }
@@ -596,6 +597,7 @@ serve = [
   "component-model",
   "dep:http-body-util",
   "dep:http",
+  "dep:pin-project-lite",
   "wasmtime-cli-flags/async",
   "wasmtime-wasi-http?/p2",
 ]

Cargo.toml

@@ -12,7 +12,7 @@
 set -ex
 
 compile() {
-  cargo run --no-default-features --features compile,pulley,wat,gc-drc,component-model,component-model-async,debug \
+  cargo run --no-default-features --features compile,pulley,wat,gc-copying,component-model,component-model-async,debug \
     compile --target pulley64 $1 \
     -o ${1%.wat}.cwasm \
     -O memory-reservation=$((1 << 20)) \

ci/miri-provenance-test.sh

@@ -557,6 +557,11 @@ fn gen_instruction_data_impl(formats: &[Rc<InstructionFormat>], fmt: &mut Format
                                     OperandKindFields::VariableArgs => {
                                         fmtln!(fmt, "{member}: mapper.map_value_list({member}),");
                                     }
+                                    OperandKindFields::ImmValue
+                                        if field.kind.rust_type == "ir::MemFlags" =>
+                                    {
+                                        fmtln!(fmt, "{member}: mapper.map_mem_flags({member}),");
+                                    }
                                     OperandKindFields::ImmValue |
                                     OperandKindFields::ImmEnum(_) |
                                     OperandKindFields::TypeVar(_) => fmtln!(fmt, "{member},"),
@@ -1153,7 +1158,14 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo
         } else {
             let t = if op.is_immediate() {
                 let t = format!("T{}", tmpl_types.len() + 1);
-                tmpl_types.push(format!("{}: Into<{}>", t, op.kind.rust_type));
+                // For memflags, the public API type is MemFlagsData (the data),
+                // while InstructionData stores MemFlags (the entity index).
+                let api_type = if op.kind.rust_type == "ir::MemFlags" {
+                    "ir::MemFlagsData"
+                } else {
+                    op.kind.rust_type
+                };
+                tmpl_types.push(format!("{t}: Into<{api_type}>"));
                 into_args.push(op.name);
                 t
             } else {
@@ -1164,9 +1176,13 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo
         }
     }
 
-    // We need to mutate `self` if this instruction accepts a value list, or will construct
-    // BlockCall values.
-    if format.has_value_list || !block_args.is_empty() {
+    // We need to mutate `self` if this instruction accepts a value list, will construct
+    // BlockCall values, or has memflags operands (which need DFG insertion).
+    let has_memflags = inst
+        .operands_in
+        .iter()
+        .any(|op| op.kind.rust_type == "ir::MemFlags");
+    if format.has_value_list || !block_args.is_empty() || has_memflags {
         args[0].push_str("mut self");
     } else {
         args[0].push_str("self");
@@ -1221,6 +1237,17 @@ fn gen_inst_builder(inst: &Instruction, format: &InstructionFormat, fmt: &mut Fo
             fmtln!(fmt, "let {} = {}.into();", arg, arg);
         }
 
+        // Insert memflags data into the DFG to get entity indices.
+        for op in &inst.operands_in {
+            if op.kind.rust_type == "ir::MemFlags" && op.is_immediate() {
+                fmtln!(
+                    fmt,
+                    "let {0} = self.data_flow_graph_mut().mem_flags.insert({0}).unwrap();",
+                    op.name
+                );
+            }
+        }
+
         // Convert block references
         for op in block_args {
             fmtln!(

cranelift/codegen/meta/src/gen_inst.rs

@@ -61,6 +61,10 @@ fn gen_common_isle(
     for ty in others.keys() {
         fmtln!(fmt, "(type {} (primitive {}))", ty, ty);
     }
+    // Also declare the MemFlagsData type, which is the resolved form of MemFlags.
+    // MemFlags is an entity index into the DFG's MemFlagsSet, while MemFlagsData
+    // contains the actual flag bits. Backend MachInst types use MemFlagsData.
+    fmt.line("(type MemFlagsData (primitive MemFlagsData))");
     fmt.empty_line();
 
     // Generate the `enum` immediates, expanding all of the available variants

cranelift/codegen/meta/src/gen_isle.rs

@@ -161,7 +161,7 @@ impl Immediates {
                 )
             },
 
-            memflags: new_imm("flags", "ir::MemFlagsData", "Memory operation flags"),
+            memflags: new_imm("flags", "ir::MemFlags", "Memory operation flags"),
 
             trapcode: {
                 let mut trapcode_values = HashMap::new();

cranelift/codegen/meta/src/shared/immediates.rs

@@ -772,7 +772,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -792,7 +792,7 @@ pub(crate) fn define(
             &formats.store,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("x", Mem).with_doc("Value to be stored"),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
@@ -817,7 +817,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -836,7 +836,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -855,7 +855,7 @@ pub(crate) fn define(
             &formats.store,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("x", iExt8),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
@@ -880,7 +880,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -899,7 +899,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -918,7 +918,7 @@ pub(crate) fn define(
             &formats.store,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("x", iExt16),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
@@ -943,7 +943,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -962,7 +962,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -981,7 +981,7 @@ pub(crate) fn define(
             &formats.store,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("x", iExt32),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
@@ -1069,7 +1069,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -1087,7 +1087,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -1115,7 +1115,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -1133,7 +1133,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -1161,7 +1161,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -1179,7 +1179,7 @@ pub(crate) fn define(
             &formats.load,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"),
         ])
@@ -3132,17 +3132,17 @@ pub(crate) fn define(
 
         The input and output types must be storable to memory and of the same
         size. A bitcast is equivalent to storing one type and loading the other
-        type from the same address, both using the specified MemFlagsData.
+        type from the same address, both using the specified MemFlags.
 
-        Note that this operation only supports the `big` or `little` MemFlagsData.
+        Note that this operation only supports the `big` or `little` MemFlags.
         The specified byte order only affects the result in the case where
         input and output types differ in lane count/size.  In this case, the
         operation is only valid if a byte order specifier is provided.
         "#,
             &formats.load_no_offset,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("x", Mem),
         ])
         .operands_out(vec![
@@ -3773,7 +3773,7 @@ pub(crate) fn define(
             &formats.atomic_rmw,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("AtomicRmwOp", &imm.atomic_rmw_op),
             Operand::new("p", iAddr),
             Operand::new("x", AtomicMem).with_doc("Value to be atomically stored"),
@@ -3802,7 +3802,7 @@ pub(crate) fn define(
             &formats.atomic_cas,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
             Operand::new("e", AtomicMem).with_doc("Expected value in CAS"),
             Operand::new("x", AtomicMem).with_doc("Value to be atomically stored"),
@@ -3830,7 +3830,7 @@ pub(crate) fn define(
             &formats.load_no_offset,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("p", iAddr),
         ])
         .operands_out(vec![
@@ -3855,7 +3855,7 @@ pub(crate) fn define(
             &formats.store_no_offset,
         )
         .operands_in(vec![
-            Operand::new("MemFlagsData", &imm.memflags),
+            Operand::new("MemFlags", &imm.memflags),
             Operand::new("x", AtomicMem).with_doc("Value to be atomically stored"),
             Operand::new("p", iAddr),
         ])