From c3325dc909153da4c5499515159490335be05231 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Sun, 6 Feb 2022 09:20:19 +0000 Subject: [PATCH 1/2] Fix run-docker.sh so it can be run locally --- ci/run-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 4bb2a78d..8c4af0ef 100755 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -19,7 +19,7 @@ run() { -e CARGO_HOME=/cargo \ -e CARGO_TARGET_DIR=/target \ -e RUST_COMPILER_RT_ROOT \ - -v $(dirname $(dirname `which cargo`)):/cargo \ + -v "${HOME}/.cargo":/cargo \ -v `pwd`/target:/target \ -v `pwd`:/checkout:ro \ -v `rustc --print sysroot`:/rust:ro \ From fe04b8d4df2329d7ab8c7366f33e1f8572d51d64 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Sun, 6 Feb 2022 09:20:43 +0000 Subject: [PATCH 2/2] Wrap all intrinsics in the intrinsics! macro This ensures that each intrinsic ends up in a separate module, which in turn (because rustc treats compiler_builtins specially) will result in each intrinsic ending up in its own object file. This allows the linker to only pick up object files for intrinsics that are missing and avoids duplicate symbol definition errors. --- src/arm.rs | 376 +++++++++++++++++++---------------------------- src/arm_linux.rs | 21 +-- src/macros.rs | 119 ++++++++++++--- src/mem/mod.rs | 112 +++++++------- src/x86.rs | 148 +++++++++---------- src/x86_64.rs | 158 ++++++++++---------- 6 files changed, 469 insertions(+), 465 deletions(-) diff --git a/src/arm.rs b/src/arm.rs index 3660825a..95bde511 100644 --- a/src/arm.rs +++ b/src/arm.rs @@ -3,251 +3,181 @@ use core::intrinsics; -// NOTE This function and the ones below are implemented using assembly because they are using a -// custom calling convention which can't be implemented using a normal Rust function. -// NOTE The only difference between the iOS and non-iOS versions of those functions is that the iOS -// versions use 3 leading underscores in the names of called functions instead of 2. -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_uidivmod() { - core::arch::asm!( - "push {{lr}}", - "sub sp, sp, #4", - "mov r2, sp", - "bl __udivmodsi4", - "ldr r1, [sp]", - "add sp, sp, #4", - "pop {{pc}}", - options(noreturn) - ); -} - +// iOS symbols have a leading underscore. #[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_uidivmod() { - core::arch::asm!( - "push {{lr}}", - "sub sp, sp, #4", - "mov r2, sp", - "bl ___udivmodsi4", - "ldr r1, [sp]", - "add sp, sp, #4", - "pop {{pc}}", - options(noreturn) - ); +macro_rules! bl { + ($func:literal) => { + concat!("bl _", $func) + }; } - #[cfg(not(target_os = "ios"))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_uldivmod() { - core::arch::asm!( - "push {{r4, lr}}", - "sub sp, sp, #16", - "add r4, sp, #8", - "str r4, [sp]", - "bl __udivmoddi4", - "ldr r2, [sp, #8]", - "ldr r3, [sp, #12]", - "add sp, sp, #16", - "pop {{r4, pc}}", - options(noreturn) - ); -} - -#[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_uldivmod() { - core::arch::asm!( - "push {{r4, lr}}", - "sub sp, sp, #16", - "add r4, sp, #8", - "str r4, [sp]", - "bl ___udivmoddi4", - "ldr r2, [sp, #8]", - "ldr r3, [sp, #12]", - "add sp, sp, #16", - "pop {{r4, pc}}", - options(noreturn) - ); -} - -#[cfg(not(target_os = "ios"))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_idivmod() { - core::arch::asm!( - "push {{r0, r1, r4, lr}}", - "bl __aeabi_idiv", - "pop {{r1, r2}}", - "muls r2, r2, r0", - "subs r1, r1, r2", - "pop {{r4, pc}}", - options(noreturn) - ); -} - -#[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_idivmod() { - core::arch::asm!( - "push {{r0, r1, r4, lr}}", - "bl ___aeabi_idiv", - "pop {{r1, r2}}", - "muls r2, r2, r0", - "subs r1, r1, r2", - "pop {{r4, pc}}", - options(noreturn) - ); -} +macro_rules! bl { + ($func:literal) => { + concat!("bl ", $func) + }; +} + +intrinsics! { + // NOTE This function and the ones below are implemented using assembly because they are using a + // custom calling convention which can't be implemented using a normal Rust function. + #[naked] + #[cfg(not(target_env = "msvc"))] + pub unsafe extern "C" fn __aeabi_uidivmod() { + core::arch::asm!( + "push {{lr}}", + "sub sp, sp, #4", + "mov r2, sp", + bl!("__udivmodsi4"), + "ldr r1, [sp]", + "add sp, sp, #4", + "pop {{pc}}", + options(noreturn) + ); + } -#[cfg(not(target_os = "ios"))] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_ldivmod() { - core::arch::asm!( - "push {{r4, lr}}", - "sub sp, sp, #16", - "add r4, sp, #8", - "str r4, [sp]", - "bl __divmoddi4", - "ldr r2, [sp, #8]", - "ldr r3, [sp, #12]", - "add sp, sp, #16", - "pop {{r4, pc}}", - options(noreturn) - ); -} + #[naked] + pub unsafe extern "C" fn __aeabi_uldivmod() { + core::arch::asm!( + "push {{r4, lr}}", + "sub sp, sp, #16", + "add r4, sp, #8", + "str r4, [sp]", + bl!("__udivmodsi4"), + "ldr r2, [sp, #8]", + "ldr r3, [sp, #12]", + "add sp, sp, #16", + "pop {{r4, pc}}", + options(noreturn) + ); + } -#[cfg(target_os = "ios")] -#[naked] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __aeabi_ldivmod() { - core::arch::asm!( - "push {{r4, lr}}", - "sub sp, sp, #16", - "add r4, sp, #8", - "str r4, [sp]", - "bl ___divmoddi4", - "ldr r2, [sp, #8]", - "ldr r3, [sp, #12]", - "add sp, sp, #16", - "pop {{r4, pc}}", - options(noreturn) - ); -} + #[naked] + pub unsafe extern "C" fn __aeabi_idivmod() { + core::arch::asm!( + "push {{r0, r1, r4, lr}}", + bl!("__aeabi_idiv"), + "pop {{r1, r2}}", + "muls r2, r2, r0", + "subs r1, r1, r2", + "pop {{r4, pc}}", + options(noreturn) + ); + } -// The following functions use weak linkage to allow users to override -// with custom implementation. -// FIXME: The `*4` and `*8` variants should be defined as aliases. + #[naked] + pub unsafe extern "C" fn __aeabi_ldivmod() { + core::arch::asm!( + "push {{r4, lr}}", + "sub sp, sp, #16", + "add r4, sp, #8", + "str r4, [sp]", + bl!("__divmoddi4"), + "ldr r2, [sp, #8]", + "ldr r3, [sp, #12]", + "add sp, sp, #16", + "pop {{r4, pc}}", + options(noreturn) + ); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) { - ::mem::memcpy(dest, src, n); -} + // The following functions use weak linkage to allow users to override + // with custom implementation. + // FIXME: The `*4` and `*8` variants should be defined as aliases. -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, mut n: usize) { - // We are guaranteed 4-alignment, so accessing at u32 is okay. - let mut dest = dest as *mut u32; - let mut src = src as *mut u32; + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memcpy(dest: *mut u8, src: *const u8, n: usize) { + ::mem::memcpy(dest, src, n); + } - while n >= 4 { - *dest = *src; - dest = dest.offset(1); - src = src.offset(1); - n -= 4; + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memcpy4(dest: *mut u8, src: *const u8, n: usize) { + // We are guaranteed 4-alignment, so accessing at u32 is okay. + let mut dest = dest as *mut u32; + let mut src = src as *mut u32; + let mut n = n; + + while n >= 4 { + *dest = *src; + dest = dest.offset(1); + src = src.offset(1); + n -= 4; + } + + __aeabi_memcpy(dest as *mut u8, src as *const u8, n); } - __aeabi_memcpy(dest as *mut u8, src as *const u8, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) { + __aeabi_memcpy4(dest, src, n); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memcpy8(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memcpy4(dest, src, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) { + ::mem::memmove(dest, src, n); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memmove(dest: *mut u8, src: *const u8, n: usize) { - ::mem::memmove(dest, src, n); -} + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) { + __aeabi_memmove(dest, src, n); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memmove4(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memmove(dest, src, n); -} + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) { + __aeabi_memmove(dest, src, n); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memmove8(dest: *mut u8, src: *const u8, n: usize) { - __aeabi_memmove(dest, src, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) { + // Note the different argument order + ::mem::memset(dest, c, n); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memset(dest: *mut u8, n: usize, c: i32) { - // Note the different argument order - ::mem::memset(dest, c, n); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, n: usize, c: i32) { + let mut dest = dest as *mut u32; + let mut n = n; -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memset4(dest: *mut u8, mut n: usize, c: i32) { - let mut dest = dest as *mut u32; + let byte = (c as u32) & 0xff; + let c = (byte << 24) | (byte << 16) | (byte << 8) | byte; - let byte = (c as u32) & 0xff; - let c = (byte << 24) | (byte << 16) | (byte << 8) | byte; + while n >= 4 { + *dest = c; + dest = dest.offset(1); + n -= 4; + } - while n >= 4 { - *dest = c; - dest = dest.offset(1); - n -= 4; + __aeabi_memset(dest as *mut u8, n, byte as i32); } - __aeabi_memset(dest as *mut u8, n, byte as i32); -} - -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) { - __aeabi_memset4(dest, n, c); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memset8(dest: *mut u8, n: usize, c: i32) { + __aeabi_memset4(dest, n, c); + } -#[cfg(not(target_os = "ios"))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) { - __aeabi_memset(dest, n, 0); -} + #[cfg(not(target_os = "ios"))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memclr(dest: *mut u8, n: usize) { + __aeabi_memset(dest, n, 0); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) { - __aeabi_memset4(dest, n, 0); -} + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memclr4(dest: *mut u8, n: usize) { + __aeabi_memset4(dest, n, 0); + } -#[cfg(not(any(target_os = "ios", target_env = "msvc")))] -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -#[linkage = "weak"] -pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) { - __aeabi_memset4(dest, n, 0); + #[cfg(not(any(target_os = "ios", target_env = "msvc")))] + #[linkage = "weak"] + pub unsafe extern "aapcs" fn __aeabi_memclr8(dest: *mut u8, n: usize) { + __aeabi_memset4(dest, n, 0); + } } diff --git a/src/arm_linux.rs b/src/arm_linux.rs index e710c1ab..df1723d9 100644 --- a/src/arm_linux.rs +++ b/src/arm_linux.rs @@ -90,17 +90,19 @@ unsafe fn atomic_cmpxchg(ptr: *mut T, oldval: u32, newval: u32) -> u32 { macro_rules! atomic_rmw { ($name:ident, $ty:ty, $op:expr) => { - #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty { - atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty + intrinsics! { + pub unsafe extern "C" fn $name(ptr: *mut $ty, val: $ty) -> $ty { + atomic_rmw(ptr, |x| $op(x as $ty, val) as u32) as $ty + } } }; } macro_rules! atomic_cmpxchg { ($name:ident, $ty:ty) => { - #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty { - atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty + intrinsics! { + pub unsafe extern "C" fn $name(ptr: *mut $ty, oldval: $ty, newval: $ty) -> $ty { + atomic_cmpxchg(ptr, oldval as u32, newval as u32) as $ty + } } }; } @@ -205,7 +207,8 @@ atomic_cmpxchg!(__sync_val_compare_and_swap_1, u8); atomic_cmpxchg!(__sync_val_compare_and_swap_2, u16); atomic_cmpxchg!(__sync_val_compare_and_swap_4, u32); -#[cfg_attr(not(feature = "mangled-names"), no_mangle)] -pub unsafe extern "C" fn __sync_synchronize() { - __kuser_memory_barrier(); +intrinsics! { + pub unsafe extern "C" fn __sync_synchronize() { + __kuser_memory_barrier(); + } } diff --git a/src/macros.rs b/src/macros.rs index 214f0795..6926feac 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -76,7 +76,7 @@ macro_rules! intrinsics { ( #[maybe_use_optimized_c_shim] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } @@ -84,9 +84,9 @@ macro_rules! intrinsics { ) => ( #[cfg($name = "optimized-c")] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { extern $abi { - fn $name($($argname: $ty),*) -> $ret; + fn $name($($argname: $ty),*) $(-> $ret)?; } unsafe { $name($($argname),*) @@ -96,7 +96,7 @@ macro_rules! intrinsics { #[cfg(not($name = "optimized-c"))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -110,7 +110,7 @@ macro_rules! intrinsics { ( #[aapcs_on_arm] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } @@ -119,7 +119,7 @@ macro_rules! intrinsics { #[cfg(target_arch = "arm")] intrinsics! { $(#[$($attr)*])* - pub extern "aapcs" fn $name( $($argname: $ty),* ) -> $ret { + pub extern "aapcs" fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -127,7 +127,7 @@ macro_rules! intrinsics { #[cfg(not(target_arch = "arm"))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -140,7 +140,7 @@ macro_rules! intrinsics { ( #[unadjusted_on_win64] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } @@ -149,7 +149,7 @@ macro_rules! intrinsics { #[cfg(all(windows, target_pointer_width = "64"))] intrinsics! { $(#[$($attr)*])* - pub extern "unadjusted" fn $name( $($argname: $ty),* ) -> $ret { + pub extern "unadjusted" fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -157,7 +157,7 @@ macro_rules! intrinsics { #[cfg(not(all(windows, target_pointer_width = "64")))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -175,7 +175,7 @@ macro_rules! intrinsics { ( #[win64_128bit_abi_hack] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } @@ -183,7 +183,7 @@ macro_rules! intrinsics { ) => ( #[cfg(all(windows, target_arch = "x86_64"))] $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } @@ -193,7 +193,7 @@ macro_rules! intrinsics { pub extern $abi fn $name( $($argname: $ty),* ) -> ::macros::win64_128bit_abi_hack::U64x2 { - let e: $ret = super::$name($($argname),*); + let e: $($ret)? = super::$name($($argname),*); ::macros::win64_128bit_abi_hack::U64x2::from(e) } } @@ -201,7 +201,7 @@ macro_rules! intrinsics { #[cfg(not(all(windows, target_arch = "x86_64")))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -216,21 +216,21 @@ macro_rules! intrinsics { ( #[arm_aeabi_alias = $alias:ident] $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } $($rest:tt)* ) => ( #[cfg(target_arch = "arm")] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } #[cfg(target_arch = "arm")] pub mod $name { #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { super::$name($($argname),*) } } @@ -238,7 +238,7 @@ macro_rules! intrinsics { #[cfg(target_arch = "arm")] pub mod $alias { #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub extern "aapcs" fn $alias( $($argname: $ty),* ) -> $ret { + pub extern "aapcs" fn $alias( $($argname: $ty),* ) $(-> $ret)? { super::$name($($argname),*) } } @@ -246,7 +246,57 @@ macro_rules! intrinsics { #[cfg(not(target_arch = "arm"))] intrinsics! { $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + $($body)* + } + } + + intrinsics!($($rest)*); + ); + + // C mem* functions are only generated when the "mem" feature is enabled. + ( + #[mem_builtin] + $(#[$($attr:tt)*])* + pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + + $($rest:tt)* + ) => ( + $(#[$($attr)*])* + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + $($body)* + } + + #[cfg(feature = "mem")] + pub mod $name { + $(#[$($attr)*])* + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + super::$name($($argname),*) + } + } + + intrinsics!($($rest)*); + ); + + // Naked functions are special: we can't generate wrappers for them since + // they use a custom calling convention. + ( + #[naked] + $(#[$($attr:tt)*])* + pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + + $($rest:tt)* + ) => ( + pub mod $name { + #[naked] + $(#[$($attr)*])* + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } } @@ -268,21 +318,46 @@ macro_rules! intrinsics { // input we were given. ( $(#[$($attr:tt)*])* - pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) -> $ret:ty { + pub extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { + $($body:tt)* + } + + $($rest:tt)* + ) => ( + $(#[$($attr)*])* + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + $($body)* + } + + pub mod $name { + $(#[$($attr)*])* + #[cfg_attr(not(feature = "mangled-names"), no_mangle)] + pub extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { + super::$name($($argname),*) + } + } + + intrinsics!($($rest)*); + ); + + // Same as the above for unsafe functions. + ( + $(#[$($attr:tt)*])* + pub unsafe extern $abi:tt fn $name:ident( $($argname:ident: $ty:ty),* ) $(-> $ret:ty)? { $($body:tt)* } $($rest:tt)* ) => ( $(#[$($attr)*])* - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { $($body)* } pub mod $name { $(#[$($attr)*])* #[cfg_attr(not(feature = "mangled-names"), no_mangle)] - pub extern $abi fn $name( $($argname: $ty),* ) -> $ret { + pub unsafe extern $abi fn $name( $($argname: $ty),* ) $(-> $ret)? { super::$name($($argname),*) } } diff --git a/src/mem/mod.rs b/src/mem/mod.rs index 2f9a9fd9..dce4d87e 100644 --- a/src/mem/mod.rs +++ b/src/mem/mod.rs @@ -19,53 +19,55 @@ use core::ops::{BitOr, Shl}; )] mod impls; -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] -pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { - impls::copy_forward(dest, src, n); - dest -} - -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] -pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { - let delta = (dest as usize).wrapping_sub(src as usize); - if delta >= n { - // We can copy forwards because either dest is far enough ahead of src, - // or src is ahead of dest (and delta overflowed). +intrinsics! { + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { impls::copy_forward(dest, src, n); - } else { - impls::copy_backward(dest, src, n); + dest } - dest -} -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] -pub unsafe extern "C" fn memset(s: *mut u8, c: c_int, n: usize) -> *mut u8 { - impls::set_bytes(s, c as u8, n); - s -} + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { + let delta = (dest as usize).wrapping_sub(src as usize); + if delta >= n { + // We can copy forwards because either dest is far enough ahead of src, + // or src is ahead of dest (and delta overflowed). + impls::copy_forward(dest, src, n); + } else { + impls::copy_backward(dest, src, n); + } + dest + } -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] -pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { - let mut i = 0; - while i < n { - let a = *s1.add(i); - let b = *s2.add(i); - if a != b { - return a as i32 - b as i32; + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memset(s: *mut u8, c: crate::mem::c_int, n: usize) -> *mut u8 { + impls::set_bytes(s, c as u8, n); + s + } + + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { + let mut i = 0; + while i < n { + let a = *s1.add(i); + let b = *s2.add(i); + if a != b { + return a as i32 - b as i32; + } + i += 1; } - i += 1; + 0 } - 0 -} -#[cfg_attr(all(feature = "mem", not(feature = "mangled-names")), no_mangle)] -#[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] -pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { - memcmp(s1, s2, n) + #[mem_builtin] + #[cfg_attr(not(all(target_os = "windows", target_env = "gnu")), linkage = "weak")] + pub unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { + memcmp(s1, s2, n) + } } // `bytes` must be a multiple of `mem::size_of::()` @@ -133,65 +135,65 @@ where intrinsics! { #[cfg(target_has_atomic_load_store = "8")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { memcpy_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "16")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { memcpy_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "32")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { memcpy_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "64")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { memcpy_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "128")] - pub extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memcpy_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { memcpy_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "8")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_1(dest: *mut u8, src: *const u8, bytes: usize) -> () { memmove_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "16")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_2(dest: *mut u16, src: *const u16, bytes: usize) -> () { memmove_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "32")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_4(dest: *mut u32, src: *const u32, bytes: usize) -> () { memmove_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "64")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_8(dest: *mut u64, src: *const u64, bytes: usize) -> () { memmove_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "128")] - pub extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memmove_element_unordered_atomic_16(dest: *mut u128, src: *const u128, bytes: usize) -> () { memmove_element_unordered_atomic(dest, src, bytes); } #[cfg(target_has_atomic_load_store = "8")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_1(s: *mut u8, c: u8, bytes: usize) -> () { memset_element_unordered_atomic(s, c, bytes); } #[cfg(target_has_atomic_load_store = "16")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_2(s: *mut u16, c: u8, bytes: usize) -> () { memset_element_unordered_atomic(s, c, bytes); } #[cfg(target_has_atomic_load_store = "32")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_4(s: *mut u32, c: u8, bytes: usize) -> () { memset_element_unordered_atomic(s, c, bytes); } #[cfg(target_has_atomic_load_store = "64")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_8(s: *mut u64, c: u8, bytes: usize) -> () { memset_element_unordered_atomic(s, c, bytes); } #[cfg(target_has_atomic_load_store = "128")] - pub extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () { + pub unsafe extern "C" fn __llvm_memset_element_unordered_atomic_16(s: *mut u128, c: u8, bytes: usize) -> () { memset_element_unordered_atomic(s, c, bytes); } } diff --git a/src/x86.rs b/src/x86.rs index abcc2bdb..fd1f32e3 100644 --- a/src/x86.rs +++ b/src/x86.rs @@ -8,82 +8,78 @@ use core::intrinsics; // NOTE These functions are never mangled as they are not tested against compiler-rt // and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn ___chkstk_ms() { - core::arch::asm!( - "push %ecx", - "push %eax", - "cmp $0x1000,%eax", - "lea 12(%esp),%ecx", - "jb 1f", - "2:", - "sub $0x1000,%ecx", - "test %ecx,(%ecx)", - "sub $0x1000,%eax", - "cmp $0x1000,%eax", - "ja 2b", - "1:", - "sub %eax,%ecx", - "test %ecx,(%ecx)", - "pop %eax", - "pop %ecx", - "ret", - options(noreturn, att_syntax) - ); -} +intrinsics! { + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk_ms() { + core::arch::asm!( + "push %ecx", + "push %eax", + "cmp $0x1000,%eax", + "lea 12(%esp),%ecx", + "jb 1f", + "2:", + "sub $0x1000,%ecx", + "test %ecx,(%ecx)", + "sub $0x1000,%eax", + "cmp $0x1000,%eax", + "ja 2b", + "1:", + "sub %eax,%ecx", + "test %ecx,(%ecx)", + "pop %eax", + "pop %ecx", + "ret", + options(noreturn, att_syntax) + ); + } -// FIXME: __alloca should be an alias to __chkstk -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn __alloca() { - core::arch::asm!( - "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable" - options(noreturn, att_syntax) - ); -} + // FIXME: __alloca should be an alias to __chkstk + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn __alloca() { + core::arch::asm!( + "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable" + options(noreturn, att_syntax) + ); + } -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn ___chkstk() { - core::arch::asm!( - "push %ecx", - "cmp $0x1000,%eax", - "lea 8(%esp),%ecx", // esp before calling this routine -> ecx - "jb 1f", - "2:", - "sub $0x1000,%ecx", - "test %ecx,(%ecx)", - "sub $0x1000,%eax", - "cmp $0x1000,%eax", - "ja 2b", - "1:", - "sub %eax,%ecx", - "test %ecx,(%ecx)", - "lea 4(%esp),%eax", // load pointer to the return address into eax - "mov %ecx,%esp", // install the new top of stack pointer into esp - "mov -4(%eax),%ecx", // restore ecx - "push (%eax)", // push return address onto the stack - "sub %esp,%eax", // restore the original value in eax - "ret", - options(noreturn, att_syntax) - ); + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk() { + core::arch::asm!( + "push %ecx", + "cmp $0x1000,%eax", + "lea 8(%esp),%ecx", // esp before calling this routine -> ecx + "jb 1f", + "2:", + "sub $0x1000,%ecx", + "test %ecx,(%ecx)", + "sub $0x1000,%eax", + "cmp $0x1000,%eax", + "ja 2b", + "1:", + "sub %eax,%ecx", + "test %ecx,(%ecx)", + "lea 4(%esp),%eax", // load pointer to the return address into eax + "mov %ecx,%esp", // install the new top of stack pointer into esp + "mov -4(%eax),%ecx", // restore ecx + "push (%eax)", // push return address onto the stack + "sub %esp,%eax", // restore the original value in eax + "ret", + options(noreturn, att_syntax) + ); + } } diff --git a/src/x86_64.rs b/src/x86_64.rs index ea3c9949..393eeddd 100644 --- a/src/x86_64.rs +++ b/src/x86_64.rs @@ -8,89 +8,87 @@ use core::intrinsics; // NOTE These functions are never mangled as they are not tested against compiler-rt // and mangling ___chkstk would break the `jmp ___chkstk` instruction in __alloca -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn ___chkstk_ms() { - core::arch::asm!( - "push %rcx", - "push %rax", - "cmp $0x1000,%rax", - "lea 24(%rsp),%rcx", - "jb 1f", - "2:", - "sub $0x1000,%rcx", - "test %rcx,(%rcx)", - "sub $0x1000,%rax", - "cmp $0x1000,%rax", - "ja 2b", - "1:", - "sub %rax,%rcx", - "test %rcx,(%rcx)", - "pop %rax", - "pop %rcx", - "ret", - options(noreturn, att_syntax) - ); -} +intrinsics! { + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk_ms() { + core::arch::asm!( + "push %rcx", + "push %rax", + "cmp $0x1000,%rax", + "lea 24(%rsp),%rcx", + "jb 1f", + "2:", + "sub $0x1000,%rcx", + "test %rcx,(%rcx)", + "sub $0x1000,%rax", + "cmp $0x1000,%rax", + "ja 2b", + "1:", + "sub %rax,%rcx", + "test %rcx,(%rcx)", + "pop %rax", + "pop %rcx", + "ret", + options(noreturn, att_syntax) + ); + } -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn __alloca() { - core::arch::asm!( - "mov %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx - "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable" - options(noreturn, att_syntax) - ); -} + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn __alloca() { + core::arch::asm!( + "mov %rcx,%rax", // x64 _alloca is a normal function with parameter in rcx + "jmp ___chkstk", // Jump to ___chkstk since fallthrough may be unreliable" + options(noreturn, att_syntax) + ); + } -#[cfg(all( - windows, - target_env = "gnu", - not(feature = "no-asm"), - not(feature = "mangled-names") -))] -#[naked] -#[no_mangle] -pub unsafe extern "C" fn ___chkstk() { - core::arch::asm!( - "push %rcx", - "cmp $0x1000,%rax", - "lea 16(%rsp),%rcx", // rsp before calling this routine -> rcx - "jb 1f", - "2:", - "sub $0x1000,%rcx", - "test %rcx,(%rcx)", - "sub $0x1000,%rax", - "cmp $0x1000,%rax", - "ja 2b", - "1:", - "sub %rax,%rcx", - "test %rcx,(%rcx)", - "lea 8(%rsp),%rax", // load pointer to the return address into rax - "mov %rcx,%rsp", // install the new top of stack pointer into rsp - "mov -8(%rax),%rcx", // restore rcx - "push (%rax)", // push return address onto the stack - "sub %rsp,%rax", // restore the original value in rax - "ret", - options(noreturn, att_syntax) - ); + #[naked] + #[cfg(all( + windows, + target_env = "gnu", + not(feature = "no-asm") + ))] + pub unsafe extern "C" fn ___chkstk() { + core::arch::asm!( + "push %rcx", + "cmp $0x1000,%rax", + "lea 16(%rsp),%rcx", // rsp before calling this routine -> rcx + "jb 1f", + "2:", + "sub $0x1000,%rcx", + "test %rcx,(%rcx)", + "sub $0x1000,%rax", + "cmp $0x1000,%rax", + "ja 2b", + "1:", + "sub %rax,%rcx", + "test %rcx,(%rcx)", + "lea 8(%rsp),%rax", // load pointer to the return address into rax + "mov %rcx,%rsp", // install the new top of stack pointer into rsp + "mov -8(%rax),%rcx", // restore rcx + "push (%rax)", // push return address onto the stack + "sub %rsp,%rax", // restore the original value in rax + "ret", + options(noreturn, att_syntax) + ); + } } // HACK(https://github.com/rust-lang/rust/issues/62785): x86_64-unknown-uefi needs special LLVM // support unless we emit the _fltused -#[no_mangle] -#[used] -#[cfg(target_os = "uefi")] -static _fltused: i32 = 0; +mod _fltused { + #[no_mangle] + #[used] + #[cfg(target_os = "uefi")] + static _fltused: i32 = 0; +}