Migrate libcalls to checking for traps

This commit is the equivalent of #9675 for libcalls used in core wasm.
All libcalls now communicate whether or not they trapped through their
return value instead of implicitly calling `longjmp` to exit from the
libcall. This is to make integration with Pulley easier to avoid the
need to `longjmp` over Pulley execution.

Libcall definitions have changed where appropriate and the
`catch_unwind_and_record_trap` function introduced in #9675 was
refactored to better support multiple types of values being returned
from libcalls (instead of just `Result<()>`).

Note that changes have been made to both the Cranelift translation layer
and the Winch translation layer for this as the ABI of various libcalls
are all changing.

crates/cranelift/src/compiler.rs

@@ -7,6 +7,7 @@ use crate::{builder::LinkOptions, wasm_call_signature, BuiltinFunctionSignatures
 use anyhow::{Context as _, Result};
 use cranelift_codegen::binemit::CodeOffset;
 use cranelift_codegen::bitset::CompoundBitSet;
+use cranelift_codegen::ir::condcodes::IntCC;
 use cranelift_codegen::ir::{self, InstBuilder, MemFlags, UserExternalName, UserFuncName, Value};
 use cranelift_codegen::isa::{
     unwind::{UnwindInfo, UnwindInfoKind},
@@ -28,8 +29,8 @@ use wasmparser::{FuncValidatorAllocations, FunctionBody};
 use wasmtime_environ::{
     AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, DefinedFuncIndex, FlagValue,
     FunctionBodyData, FunctionLoc, ModuleTranslation, ModuleTypesBuilder, PtrSize,
-    RelocationTarget, StackMapInformation, StaticModuleIndex, TrapEncodingBuilder, Tunables,
-    VMOffsets, WasmFuncType, WasmFunctionInfo, WasmValType,
+    RelocationTarget, StackMapInformation, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel,
+    Tunables, VMOffsets, WasmFuncType, WasmFunctionInfo, WasmValType,
 };
 
 #[cfg(feature = "component-model")]
@@ -69,7 +70,8 @@ pub struct Compiler {
     linkopts: LinkOptions,
     cache_store: Option<Arc<dyn CacheStore>>,
     clif_dir: Option<path::PathBuf>,
-    wmemcheck: bool,
+    #[cfg(feature = "wmemcheck")]
+    pub(crate) wmemcheck: bool,
 }
 
 impl Drop for Compiler {
@@ -109,13 +111,15 @@ impl Compiler {
         clif_dir: Option<path::PathBuf>,
         wmemcheck: bool,
     ) -> Compiler {
+        let _ = wmemcheck;
         Compiler {
             contexts: Default::default(),
             tunables,
             isa,
             linkopts,
             cache_store,
             clif_dir,
+            #[cfg(feature = "wmemcheck")]
             wmemcheck,
         }
     }
@@ -227,14 +231,7 @@ impl wasmtime_environ::Compiler for Compiler {
             context.func.collect_debug_info();
         }
 
-        let mut func_env = FuncEnvironment::new(
-            isa,
-            translation,
-            types,
-            &self.tunables,
-            self.wmemcheck,
-            wasm_func_ty,
-        );
+        let mut func_env = FuncEnvironment::new(self, translation, types, wasm_func_ty);
 
         // The `stack_limit` global value below is the implementation of stack
         // overflow checks in Wasmtime.
@@ -604,7 +601,7 @@ impl wasmtime_environ::Compiler for Compiler {
         let isa = &*self.isa;
         let ptr_size = isa.pointer_bytes();
         let pointer_type = isa.pointer_type();
-        let sigs = BuiltinFunctionSignatures::new(isa, &self.tunables);
+        let sigs = BuiltinFunctionSignatures::new(self);
         let wasm_sig = sigs.wasm_signature(index);
         let host_sig = sigs.host_signature(index);
 
@@ -626,11 +623,46 @@ impl wasmtime_environ::Compiler for Compiler {
         save_last_wasm_exit_fp_and_pc(&mut builder, pointer_type, &ptr_size, limits);
 
         // Now it's time to delegate to the actual builtin. Forward all our own
-        // arguments to the libcall itself, and then return all the same results
-        // as the libcall.
+        // arguments to the libcall itself.
         let args = builder.block_params(block0).to_vec();
         let call = self.call_builtin(&mut builder, vmctx, &args, index, host_sig);
         let results = builder.func.dfg.inst_results(call).to_vec();
+
+        // Libcalls do not explicitly `longjmp` for example but instead return a
+        // code indicating whether they trapped or not. This means that it's the
+        // responsibility of the trampoline to check for an trapping return
+        // value and raise a trap as appropriate. With the `results` above check
+        // what `index` is and for each libcall that has a trapping return value
+        // process it here.
+        match index.trap_sentinel() {
+            Some(TrapSentinel::Falsy) => {
+                self.raise_if_host_trapped(&mut builder, vmctx, results[0]);
+            }
+            Some(TrapSentinel::NegativeTwo) => {
+                let ty = builder.func.dfg.value_type(results[0]);
+                let trapped = builder.ins().iconst(ty, -2);
+                let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], trapped);
+                self.raise_if_host_trapped(&mut builder, vmctx, succeeded);
+            }
+            Some(TrapSentinel::Negative) => {
+                let ty = builder.func.dfg.value_type(results[0]);
+                let zero = builder.ins().iconst(ty, 0);
+                let succeeded =
+                    builder
+                        .ins()
+                        .icmp(IntCC::SignedGreaterThanOrEqual, results[0], zero);
+                self.raise_if_host_trapped(&mut builder, vmctx, succeeded);
+            }
+            Some(TrapSentinel::NegativeOne) => {
+                let ty = builder.func.dfg.value_type(results[0]);
+                let minus_one = builder.ins().iconst(ty, -1);
+                let succeeded = builder.ins().icmp(IntCC::NotEqual, results[0], minus_one);
+                self.raise_if_host_trapped(&mut builder, vmctx, succeeded);
+            }
+            None => {}
+        }
+
+        // And finally, return all the results of this libcall.
         builder.ins().return_(&results);
         builder.finalize();
 
@@ -863,7 +895,7 @@ impl Compiler {
     /// Additionally in the future for pulley this will emit a special trap
     /// opcode for Pulley itself to cease interpretation and exit the
     /// interpreter.
-    fn raise_if_host_trapped(
+    pub fn raise_if_host_trapped(
         &self,
         builder: &mut FunctionBuilder<'_>,
         vmctx: ir::Value,
@@ -880,7 +912,7 @@ impl Compiler {
         builder.seal_block(continuation_block);
 
         builder.switch_to_block(trapped_block);
-        let sigs = BuiltinFunctionSignatures::new(&*self.isa, &self.tunables);
+        let sigs = BuiltinFunctionSignatures::new(self);
         let sig = sigs.host_signature(BuiltinFunctionIndex::raise());
         self.call_builtin(builder, vmctx, &[vmctx], BuiltinFunctionIndex::raise(), sig);
         builder.ins().trap(TRAP_INTERNAL_ASSERT);
@@ -920,6 +952,14 @@ impl Compiler {
         let sig = builder.func.import_signature(sig);
         self.call_indirect_host(builder, sig, func_addr, args)
     }
+
+    pub fn isa(&self) -> &dyn TargetIsa {
+        &*self.isa
+    }
+
+    pub fn tunables(&self) -> &Tunables {
+        &self.tunables
+    }
 }
 
 struct FunctionCompiler<'a> {

crates/cranelift/src/compiler/component.rs

@@ -2,6 +2,7 @@
 
 use crate::{compiler::Compiler, TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT};
 use anyhow::Result;
+use cranelift_codegen::ir::condcodes::IntCC;
 use cranelift_codegen::ir::{self, InstBuilder, MemFlags};
 use cranelift_codegen::isa::{CallConv, TargetIsa};
 use cranelift_frontend::FunctionBuilder;
@@ -97,16 +98,22 @@ impl<'a> TrampolineCompiler<'a> {
             Trampoline::ResourceRep(ty) => self.translate_resource_rep(*ty),
             Trampoline::ResourceDrop(ty) => self.translate_resource_drop(*ty),
             Trampoline::ResourceTransferOwn => {
-                self.translate_resource_libcall(host::resource_transfer_own)
+                self.translate_resource_libcall(host::resource_transfer_own, |me, rets| {
+                    rets[0] = me.raise_if_resource_trapped(rets[0]);
+                })
             }
             Trampoline::ResourceTransferBorrow => {
-                self.translate_resource_libcall(host::resource_transfer_borrow)
+                self.translate_resource_libcall(host::resource_transfer_borrow, |me, rets| {
+                    rets[0] = me.raise_if_resource_trapped(rets[0]);
+                })
             }
             Trampoline::ResourceEnterCall => {
-                self.translate_resource_libcall(host::resource_enter_call)
+                self.translate_resource_libcall(host::resource_enter_call, |_, _| {})
             }
             Trampoline::ResourceExitCall => {
-                self.translate_resource_libcall(host::resource_exit_call)
+                self.translate_resource_libcall(host::resource_exit_call, |me, rets| {
+                    me.raise_if_host_trapped(rets.pop().unwrap());
+                })
             }
         }
     }
@@ -120,7 +127,6 @@ impl<'a> TrampolineCompiler<'a> {
         let pointer_type = self.isa.pointer_type();
         let args = self.builder.func.dfg.block_params(self.block0).to_vec();
         let vmctx = args[0];
-        let caller_vmctx = args[1];
         let wasm_func_ty = self.types[self.signature].unwrap_func();
 
         // Start off by spilling all the wasm arguments into a stack slot to be
@@ -248,8 +254,7 @@ impl<'a> TrampolineCompiler<'a> {
 
         match self.abi {
             Abi::Wasm => {
-                self.compiler
-                    .raise_if_host_trapped(&mut self.builder, caller_vmctx, succeeded);
+                self.raise_if_host_trapped(succeeded);
                 // After the host function has returned the results are loaded from
                 // `values_vec_ptr` and then returned.
                 let results = self.compiler.load_values_from_array(
@@ -284,6 +289,8 @@ impl<'a> TrampolineCompiler<'a> {
         );
         self.compiler
             .call_indirect_host(&mut self.builder, host_sig, host_fn, &[vmctx, code]);
+        let succeeded = self.builder.ins().iconst(ir::types::I8, 0);
+        self.raise_if_host_trapped(succeeded);
         // debug trap in case execution actually falls through, but this
         // shouldn't ever get hit at runtime.
         self.builder.ins().trap(TRAP_INTERNAL_ASSERT);
@@ -319,6 +326,7 @@ impl<'a> TrampolineCompiler<'a> {
             self.compiler
                 .call_indirect_host(&mut self.builder, host_sig, host_fn, &host_args);
         let result = self.builder.func.dfg.inst_results(call)[0];
+        let result = self.raise_if_resource_trapped(result);
         self.abi_store_results(&[result]);
     }
 
@@ -352,6 +360,7 @@ impl<'a> TrampolineCompiler<'a> {
             self.compiler
                 .call_indirect_host(&mut self.builder, host_sig, host_fn, &host_args);
         let result = self.builder.func.dfg.inst_results(call)[0];
+        let result = self.raise_if_resource_trapped(result);
         self.abi_store_results(&[result]);
     }
 
@@ -382,6 +391,14 @@ impl<'a> TrampolineCompiler<'a> {
                 .call_indirect_host(&mut self.builder, host_sig, host_fn, &host_args);
         let should_run_destructor = self.builder.func.dfg.inst_results(call)[0];
 
+        // Immediately raise a trap if requested by the host
+        let minus_one = self.builder.ins().iconst(ir::types::I64, -1);
+        let succeeded = self
+            .builder
+            .ins()
+            .icmp(IntCC::NotEqual, should_run_destructor, minus_one);
+        self.raise_if_host_trapped(succeeded);
+
         let resource_ty = self.types[resource].ty;
         let resource_def = self
             .component
@@ -541,6 +558,7 @@ impl<'a> TrampolineCompiler<'a> {
     fn translate_resource_libcall(
         &mut self,
         get_libcall: fn(&dyn TargetIsa, &mut ir::Function) -> (ir::SigRef, u32),
+        handle_results: fn(&mut Self, &mut Vec<ir::Value>),
     ) {
         match self.abi {
             Abi::Wasm => {}
@@ -562,7 +580,8 @@ impl<'a> TrampolineCompiler<'a> {
         let call =
             self.compiler
                 .call_indirect_host(&mut self.builder, host_sig, host_fn, &host_args);
-        let results = self.builder.func.dfg.inst_results(call).to_vec();
+        let mut results = self.builder.func.dfg.inst_results(call).to_vec();
+        handle_results(self, &mut results);
         self.builder.ins().return_(&results);
     }
 
@@ -637,6 +656,29 @@ impl<'a> TrampolineCompiler<'a> {
             }
         }
     }
+
+    fn raise_if_host_trapped(&mut self, succeeded: ir::Value) {
+        let caller_vmctx = self.builder.func.dfg.block_params(self.block0)[1];
+        self.compiler
+            .raise_if_host_trapped(&mut self.builder, caller_vmctx, succeeded);
+    }
+
+    fn raise_if_transcode_trapped(&mut self, amount_copied: ir::Value) {
+        let pointer_type = self.isa.pointer_type();
+        let minus_one = self.builder.ins().iconst(pointer_type, -1);
+        let succeeded = self
+            .builder
+            .ins()
+            .icmp(IntCC::NotEqual, amount_copied, minus_one);
+        self.raise_if_host_trapped(succeeded);
+    }
+
+    fn raise_if_resource_trapped(&mut self, ret: ir::Value) -> ir::Value {
+        let minus_one = self.builder.ins().iconst(ir::types::I64, -1);
+        let succeeded = self.builder.ins().icmp(IntCC::NotEqual, ret, minus_one);
+        self.raise_if_host_trapped(succeeded);
+        self.builder.ins().ireduce(ir::types::I32, ret)
+    }
 }
 
 impl ComponentCompiler for Compiler {
@@ -806,19 +848,23 @@ impl TrampolineCompiler<'_> {
         // Like the arguments the results are fairly similar across libcalls, so
         // they're lumped into various buckets here.
         match op {
-            Transcode::Copy(_) | Transcode::Latin1ToUtf16 => {}
+            Transcode::Copy(_) | Transcode::Latin1ToUtf16 => {
+                self.raise_if_host_trapped(results[0]);
+            }
 
             Transcode::Utf8ToUtf16
             | Transcode::Utf16ToCompactProbablyUtf16
             | Transcode::Utf8ToCompactUtf16
             | Transcode::Utf16ToCompactUtf16 => {
+                self.raise_if_transcode_trapped(results[0]);
                 raw_results.push(self.cast_from_pointer(results[0], to64));
             }
 
             Transcode::Latin1ToUtf8
             | Transcode::Utf16ToUtf8
             | Transcode::Utf8ToLatin1
             | Transcode::Utf16ToLatin1 => {
+                self.raise_if_transcode_trapped(results[0]);
                 raw_results.push(self.cast_from_pointer(results[0], from64));
                 raw_results.push(self.cast_from_pointer(results[1], to64));
             }
@@ -931,6 +977,7 @@ mod host {
         (@ty $ptr:ident ptr_u8) => ($ptr);
         (@ty $ptr:ident ptr_u16) => ($ptr);
         (@ty $ptr:ident ptr_size) => ($ptr);
+        (@ty $ptr:ident bool) => (ir::types::I8);
         (@ty $ptr:ident u8) => (ir::types::I8);
         (@ty $ptr:ident u32) => (ir::types::I32);
         (@ty $ptr:ident u64) => (ir::types::I64);