add experimental support for illumos

With this change, the basic wasm tests all pass on illumos. Note the addition
of NORESERVE to mmap calls.

However:

While wasmtime appears to be functional on illumos, it is still quite slow,
particularly in the wast tests. For example, the following test:

```
cargo +beta test --test wast -- Cranelift/pooling/tests/spec_testsuite/load.wast
```

takes 0.07 seconds on Linux, but over 5 seconds on illumos. Some profiling
suggests that this is due to lock contention inside the kernel while freeing
memory, so I don't think this is a wasmtime issue. I'd like to pull some
illumos experts in to do some debugging here as time permits, but I don't think
this PR should necessarily be held up on that.

Thanks to iximeow for all the help with this!

(One note is that due to a [rustc segfault on
illumos](https://sunshowers.io/posts/rustc-segfault-illumos/), building
wasmtime requires Rust 1.83 or higher. I did my building and testing with
`+beta`.)

.github/workflows/main.yml

@@ -520,6 +520,30 @@ jobs:
       env:
         GH_TOKEN: ${{ github.token }}
 
+  checks_illumos:
+    name: Check illumos
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/actions/install-rust
+
+    # Check whether `wasmtime` cross-compiles to illumos. We need to use `cross` for this (even for
+    # cargo check) because of non-Rust dependencies. - name: Install cross
+      run: |
+        curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
+        cargo binstall --no-confirm cross
+    - name: Cross-compile to illumos
+      run:
+        cross build --target x86_64-unknown-illumos
+
+    # common logic to cancel the entire run if this job fails
+    - run: gh run cancel ${{ github.run_id }}
+      if: failure() && github.event_name != 'pull_request'
+      env:
+        GH_TOKEN: ${{ github.token }}
+
   # Check whether `wasmtime` cross-compiles to aarch64-pc-windows-msvc
   # We don't build nor test it because it lacks trap handling.
   # Tracking issue: https://github.com/bytecodealliance/wasmtime/issues/4992
@@ -1163,6 +1187,7 @@ jobs:
       - no_std_checks
       - clippy
       - monolith_checks
+      - checks_illumos
       - checks_winarm64
       - bench
       - meta_deterministic_check

crates/wasmtime/Cargo.toml

@@ -74,7 +74,7 @@ features = [
   "Win32_Security",
 ]
 
-[target.'cfg(all(target_arch = "x86_64", not(target_os = "android")))'.dependencies]
+[target.'cfg(all(target_arch = "x86_64", not(any(target_os = "android", target_os = "illumos"))))'.dependencies]
 ittapi = { version = "0.4.0", optional = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]

crates/wasmtime/src/profiling_agent.rs

@@ -29,7 +29,16 @@ cfg_if::cfg_if! {
 cfg_if::cfg_if! {
     // Note: VTune support is disabled on windows mingw because the ittapi crate doesn't compile
     // there; see also https://github.com/bytecodealliance/wasmtime/pull/4003 for rationale.
-    if #[cfg(all(feature = "profiling", target_arch = "x86_64", not(any(target_os = "android", all(target_os = "windows", target_env = "gnu")))))] {
+    if #[cfg(all(
+            feature = "profiling",
+            target_arch = "x86_64",
+            not(any(
+                target_os = "android",
+                target_os = "illumos",
+                all(target_os = "windows", target_env = "gnu"),
+            )),
+        ))
+    ] {
         mod vtune;
         pub use vtune::new as new_vtune;
     } else {

crates/wasmtime/src/runtime/vm/sys/unix/mmap.rs

@@ -11,6 +11,23 @@ pub struct Mmap {
     memory: SendSyncPtr<[u8]>,
 }
 
+cfg_if::cfg_if! {
+    if #[cfg(any(target_os = "illumos", target_os = "linux"))] {
+        // On illumos, by default, mmap reserves what it calls "swap space" ahead of time, so that
+        // memory accesses are guaranteed not to fail once mmap succeeds. NORESERVE is for cases
+        // where that memory is never meant to be accessed -- e.g. memory that's used as guard
+        // pages.
+        //
+        // This is less crucial on Linux because Linux tends to overcommit memory by default, but is
+        // still a good idea to pass in for large allocations that don't need to be backed by
+        // physical memory.
+        pub(super) const MMAP_NORESERVE_FLAG: rustix::mm::MapFlags =
+            rustix::mm::MapFlags::NORESERVE;
+    } else {
+        pub(super) const MMAP_NORESERVE_FLAG: rustix::mm::MapFlags = rustix::mm::MapFlags::empty();
+    }
+}
+
 impl Mmap {
     pub fn new_empty() -> Mmap {
         Mmap {
@@ -24,7 +41,7 @@ impl Mmap {
                 ptr::null_mut(),
                 size,
                 rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE,
-                rustix::mm::MapFlags::PRIVATE,
+                rustix::mm::MapFlags::PRIVATE | MMAP_NORESERVE_FLAG,
             )
             .err2anyhow()?
         };
@@ -39,7 +56,18 @@ impl Mmap {
                 ptr::null_mut(),
                 size,
                 rustix::mm::ProtFlags::empty(),
-                rustix::mm::MapFlags::PRIVATE,
+                // Astute readers might be wondering why a function called "reserve" passes in a
+                // NORESERVE flag. That's because "reserve" in this context means one of two
+                // different things.
+                //
+                // * This method is used to allocate virtual memory that starts off in a state where
+                //   it cannot be accessed (i.e. causes a segfault if accessed).
+                // * NORESERVE is meant for virtual memory space for which backing physical/swap
+                //   pages are reserved on first access.
+                //
+                // Virtual memory that cannot be accessed should not have a backing store reserved
+                // for it. Hence, passing in NORESERVE is correct here.
+                rustix::mm::MapFlags::PRIVATE | MMAP_NORESERVE_FLAG,
             )
             .err2anyhow()?
         };

crates/wasmtime/src/runtime/vm/sys/unix/signals.rs

@@ -279,7 +279,7 @@ pub fn abort_stack_overflow() -> ! {
                                            // help much anyway
 unsafe fn get_trap_registers(cx: *mut libc::c_void, _signum: libc::c_int) -> TrapRegisters {
     cfg_if::cfg_if! {
-        if #[cfg(all(any(target_os = "linux", target_os = "android"), target_arch = "x86_64"))] {
+        if #[cfg(all(any(target_os = "linux", target_os = "android", target_os = "illumos"), target_arch = "x86_64"))] {
             let cx = &*(cx as *const libc::ucontext_t);
             TrapRegisters {
                 pc: cx.uc_mcontext.gregs[libc::REG_RIP as usize] as usize,
@@ -347,8 +347,7 @@ unsafe fn get_trap_registers(cx: *mut libc::c_void, _signum: libc::c_int) -> Tra
                 pc: cx.sc_rip as usize,
                 fp: cx.sc_rbp as usize,
             }
-        }
-        else {
+        } else {
             compile_error!("unsupported platform");
             panic!();
         }

crates/wasmtime/src/runtime/vm/sys/unix/vm.rs

@@ -21,7 +21,7 @@ pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()>
         ptr.cast(),
         len,
         ProtFlags::empty(),
-        MapFlags::PRIVATE | MapFlags::FIXED,
+        MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
     )?;
     assert_eq!(ptr, ret.cast());
     Ok(())
@@ -56,7 +56,7 @@ pub unsafe fn decommit_pages(addr: *mut u8, len: usize) -> io::Result<()> {
                     addr as _,
                     len,
                     ProtFlags::READ | ProtFlags::WRITE,
-                    MapFlags::PRIVATE | MapFlags::FIXED,
+                    MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
                 )?;
             }
         }
@@ -176,7 +176,7 @@ impl MemoryImageSource {
             base.cast(),
             len,
             ProtFlags::READ | ProtFlags::WRITE,
-            MapFlags::PRIVATE | MapFlags::FIXED,
+            MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
         )?;
         assert_eq!(base, ptr.cast());
         Ok(())