diff --git a/src/backends/apple_other.rs b/src/backends/apple_other.rs index a8c0a24d..c7b51c0e 100644 --- a/src/backends/apple_other.rs +++ b/src/backends/apple_other.rs @@ -4,6 +4,7 @@ use core::{ffi::c_void, mem::MaybeUninit}; pub use crate::util::{inner_u32, inner_u64}; +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { let dst_ptr = dest.as_mut_ptr().cast::(); let ret = unsafe { libc::CCRandomGenerateBytes(dst_ptr, dest.len()) }; diff --git a/src/backends/custom.rs b/src/backends/custom.rs index 0c482946..c505481a 100644 --- a/src/backends/custom.rs +++ b/src/backends/custom.rs @@ -4,6 +4,7 @@ use core::mem::MaybeUninit; pub use crate::util::{inner_u32, inner_u64}; +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { extern "Rust" { fn __getrandom_v03_custom(dest: *mut u8, len: usize) -> Result<(), Error>; diff --git a/src/backends/esp_idf.rs b/src/backends/esp_idf.rs index 7ee391ab..4d1689dc 100644 --- a/src/backends/esp_idf.rs +++ b/src/backends/esp_idf.rs @@ -8,6 +8,7 @@ extern "C" { fn esp_fill_random(buf: *mut c_void, len: usize) -> u32; } +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { // Not that NOT enabling WiFi, BT, or the voltage noise entropy source (via `bootloader_random_enable`) // will cause ESP-IDF to return pseudo-random numbers based on the voltage noise entropy, after the initial boot process: diff --git a/src/backends/fuchsia.rs b/src/backends/fuchsia.rs index 5edd210d..b5f1ade5 100644 --- a/src/backends/fuchsia.rs +++ b/src/backends/fuchsia.rs @@ -9,6 +9,7 @@ extern "C" { fn zx_cprng_draw(buffer: *mut u8, length: usize); } +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { unsafe { zx_cprng_draw(dest.as_mut_ptr().cast::(), dest.len()) } Ok(()) diff --git a/src/backends/getentropy.rs b/src/backends/getentropy.rs index e0b2d34c..ed181f01 100644 --- a/src/backends/getentropy.rs +++ b/src/backends/getentropy.rs @@ -15,6 +15,7 @@ pub use crate::util::{inner_u32, inner_u64}; #[path = "../util_libc.rs"] mod util_libc; +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { for chunk in dest.chunks_mut(256) { let ret = unsafe { libc::getentropy(chunk.as_mut_ptr().cast::(), chunk.len()) }; diff --git a/src/backends/getrandom.rs b/src/backends/getrandom.rs index a00829f7..4a7b96b0 100644 --- a/src/backends/getrandom.rs +++ b/src/backends/getrandom.rs @@ -23,6 +23,7 @@ pub use crate::util::{inner_u32, inner_u64}; #[path = "../util_libc.rs"] mod util_libc; +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { util_libc::sys_fill_exact(dest, |buf| unsafe { libc::getrandom(buf.as_mut_ptr().cast::(), buf.len(), 0) diff --git a/src/backends/hermit.rs b/src/backends/hermit.rs index 11bfbf27..0c0e3a6d 100644 --- a/src/backends/hermit.rs +++ b/src/backends/hermit.rs @@ -12,6 +12,7 @@ extern "C" { fn sys_secure_rand64(value: *mut u64) -> i32; } +#[inline] pub fn inner_u32() -> Result { let mut res = MaybeUninit::uninit(); let ret = unsafe { sys_secure_rand32(res.as_mut_ptr()) }; @@ -22,6 +23,7 @@ pub fn inner_u32() -> Result { } } +#[inline] pub fn inner_u64() -> Result { let mut res = MaybeUninit::uninit(); let ret = unsafe { sys_secure_rand64(res.as_mut_ptr()) }; @@ -32,6 +34,7 @@ pub fn inner_u64() -> Result { } } +#[inline] pub fn fill_inner(mut dest: &mut [MaybeUninit]) -> Result<(), Error> { while !dest.is_empty() { let res = unsafe { sys_read_entropy(dest.as_mut_ptr().cast::(), dest.len(), 0) }; diff --git a/src/backends/linux_android.rs b/src/backends/linux_android.rs index 6c0b66ae..58b4fe80 100644 --- a/src/backends/linux_android.rs +++ b/src/backends/linux_android.rs @@ -10,6 +10,7 @@ mod util_libc; #[cfg(not(any(target_os = "android", target_os = "linux")))] compile_error!("`linux_getrandom` backend can be enabled only for Linux/Android targets!"); +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { util_libc::sys_fill_exact(dest, |buf| unsafe { libc::getrandom(buf.as_mut_ptr().cast(), buf.len(), 0) diff --git a/src/backends/linux_android_with_fallback.rs b/src/backends/linux_android_with_fallback.rs index cdfff980..33dabd2e 100644 --- a/src/backends/linux_android_with_fallback.rs +++ b/src/backends/linux_android_with_fallback.rs @@ -20,6 +20,7 @@ const NOT_AVAILABLE: NonNull = unsafe { NonNull::new_unchecked(usize::MA static GETRANDOM_FN: AtomicPtr = AtomicPtr::new(ptr::null_mut()); #[cold] +#[inline(never)] fn init() -> NonNull { static NAME: &[u8] = b"getrandom\0"; let name_ptr = NAME.as_ptr().cast::(); @@ -59,6 +60,7 @@ fn use_file_fallback(dest: &mut [MaybeUninit]) -> Result<(), Error> { use_file::fill_inner(dest) } +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { // Despite being only a single atomic variable, we still cannot always use // Ordering::Relaxed, as we need to make sure a successful call to `init` @@ -75,7 +77,7 @@ pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { if fptr == NOT_AVAILABLE { use_file_fallback(dest) } else { - // note: `transume` is currently the only way to convert pointer into function reference + // note: `transmute` is currently the only way to convert a pointer into a function reference let getrandom_fn = unsafe { mem::transmute::, GetRandomFn>(fptr) }; util_libc::sys_fill_exact(dest, |buf| unsafe { getrandom_fn(buf.as_mut_ptr().cast(), buf.len(), 0) diff --git a/src/backends/netbsd.rs b/src/backends/netbsd.rs index 57fa91a0..f228a8b1 100644 --- a/src/backends/netbsd.rs +++ b/src/backends/netbsd.rs @@ -45,6 +45,7 @@ type GetRandomFn = unsafe extern "C" fn(*mut c_void, libc::size_t, libc::c_uint) static GETRANDOM: AtomicPtr = AtomicPtr::new(ptr::null_mut()); #[cold] +#[inline(never)] fn init() -> *mut c_void { static NAME: &[u8] = b"getrandom\0"; let name_ptr = NAME.as_ptr().cast::(); @@ -58,6 +59,7 @@ fn init() -> *mut c_void { ptr } +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { // Despite being only a single atomic variable, we still cannot always use // Ordering::Relaxed, as we need to make sure a successful call to `init` diff --git a/src/backends/rdrand.rs b/src/backends/rdrand.rs index 347786a5..609fcc38 100644 --- a/src/backends/rdrand.rs +++ b/src/backends/rdrand.rs @@ -147,6 +147,7 @@ unsafe fn rdrand_u64() -> Option { Some((u64::from(a) << 32) | u64::from(b)) } +#[inline] pub fn inner_u32() -> Result { if !RDRAND_GOOD.unsync_init(is_rdrand_good) { return Err(Error::NO_RDRAND); @@ -155,6 +156,7 @@ pub fn inner_u32() -> Result { unsafe { rdrand_u32() }.ok_or(Error::FAILED_RDRAND) } +#[inline] pub fn inner_u64() -> Result { if !RDRAND_GOOD.unsync_init(is_rdrand_good) { return Err(Error::NO_RDRAND); @@ -163,6 +165,7 @@ pub fn inner_u64() -> Result { unsafe { rdrand_u64() }.ok_or(Error::FAILED_RDRAND) } +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { if !RDRAND_GOOD.unsync_init(is_rdrand_good) { return Err(Error::NO_RDRAND); diff --git a/src/backends/rndr.rs b/src/backends/rndr.rs index fa24a097..eea741a2 100644 --- a/src/backends/rndr.rs +++ b/src/backends/rndr.rs @@ -108,6 +108,7 @@ fn is_rndr_available() -> bool { } } +#[inline] pub fn inner_u32() -> Result { if !is_rndr_available() { return Err(Error::RNDR_NOT_AVAILABLE); @@ -117,6 +118,7 @@ pub fn inner_u32() -> Result { res.map(truncate).ok_or(Error::RNDR_FAILURE) } +#[inline] pub fn inner_u64() -> Result { if !is_rndr_available() { return Err(Error::RNDR_NOT_AVAILABLE); @@ -126,6 +128,7 @@ pub fn inner_u64() -> Result { res.ok_or(Error::RNDR_FAILURE) } +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { if !is_rndr_available() { return Err(Error::RNDR_NOT_AVAILABLE); diff --git a/src/backends/solaris.rs b/src/backends/solaris.rs index ea5344fc..c27f91a5 100644 --- a/src/backends/solaris.rs +++ b/src/backends/solaris.rs @@ -22,6 +22,7 @@ mod util_libc; const MAX_BYTES: usize = 1024; +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { for chunk in dest.chunks_mut(MAX_BYTES) { let ptr = chunk.as_mut_ptr().cast::(); diff --git a/src/backends/solid.rs b/src/backends/solid.rs index 6699e686..c4222a7f 100644 --- a/src/backends/solid.rs +++ b/src/backends/solid.rs @@ -8,6 +8,7 @@ extern "C" { pub fn SOLID_RNG_SampleRandomBytes(buffer: *mut u8, length: usize) -> i32; } +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { let ret = unsafe { SOLID_RNG_SampleRandomBytes(dest.as_mut_ptr().cast::(), dest.len()) }; if ret >= 0 { diff --git a/src/backends/use_file.rs b/src/backends/use_file.rs index ef12fca1..43d60b50 100644 --- a/src/backends/use_file.rs +++ b/src/backends/use_file.rs @@ -40,6 +40,7 @@ const FD_ONGOING_INIT: libc::c_int = -2; // `Ordering::Acquire` to synchronize with it. static FD: AtomicI32 = AtomicI32::new(FD_UNINIT); +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { let mut fd = FD.load(Ordering::Acquire); if fd == FD_UNINIT || fd == FD_ONGOING_INIT { @@ -77,6 +78,7 @@ fn open_readonly(path: &[u8]) -> Result { } #[cold] +#[inline(never)] fn open_or_wait() -> Result { loop { match FD.load(Ordering::Acquire) { diff --git a/src/backends/vxworks.rs b/src/backends/vxworks.rs index f03ab184..5f5e6773 100644 --- a/src/backends/vxworks.rs +++ b/src/backends/vxworks.rs @@ -11,20 +11,25 @@ mod util_libc; pub use crate::util::{inner_u32, inner_u64}; +static RNG_INIT: AtomicBool = AtomicBool::new(false); + +#[cold] +fn init() -> Result<(), Error> { + let ret = unsafe { libc::randSecure() }; + match ret.cmp(&0) { + Greater => RNG_INIT.store(true, Relaxed), + Equal => unsafe { + libc::usleep(10); + }, + Less => return Err(Error::VXWORKS_RAND_SECURE), + } + Ok(()) +} + +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { - static RNG_INIT: AtomicBool = AtomicBool::new(false); while !RNG_INIT.load(Relaxed) { - let ret = unsafe { libc::randSecure() }; - match ret.cmp(&0) { - Greater => { - RNG_INIT.store(true, Relaxed); - break; - } - Equal => unsafe { - libc::usleep(10); - }, - Less => return Err(Error::VXWORKS_RAND_SECURE), - } + init()?; } // Prevent overflow of i32 diff --git a/src/backends/wasi_p1.rs b/src/backends/wasi_p1.rs index 6eefdee6..424b7f96 100644 --- a/src/backends/wasi_p1.rs +++ b/src/backends/wasi_p1.rs @@ -11,6 +11,7 @@ extern "C" { fn random_get(arg0: i32, arg1: i32) -> i32; } +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { // Based on the wasi code: // https://docs.rs/wasi/0.11.0+wasi-snapshot-preview1/src/wasi/lib_generated.rs.html#2046-2062 diff --git a/src/backends/wasi_p2.rs b/src/backends/wasi_p2.rs index 9d5d601d..63bd2d7c 100644 --- a/src/backends/wasi_p2.rs +++ b/src/backends/wasi_p2.rs @@ -3,15 +3,18 @@ use crate::Error; use core::mem::MaybeUninit; use wasi::random::random::get_random_u64; +#[inline] pub fn inner_u32() -> Result { let val = get_random_u64(); Ok(crate::util::truncate(val)) } +#[inline] pub fn inner_u64() -> Result { Ok(get_random_u64()) } +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { use core::ptr::copy_nonoverlapping; use wasi::random::random::get_random_u64; diff --git a/src/backends/wasm_js.rs b/src/backends/wasm_js.rs index fe8f1a6e..1320d9fc 100644 --- a/src/backends/wasm_js.rs +++ b/src/backends/wasm_js.rs @@ -14,6 +14,7 @@ use wasm_bindgen::{prelude::wasm_bindgen, JsValue}; const MAX_BUFFER_SIZE: usize = 65536; #[cfg(not(target_feature = "atomics"))] +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { for chunk in dest.chunks_mut(MAX_BUFFER_SIZE) { if get_random_values(chunk).is_err() { diff --git a/src/backends/windows.rs b/src/backends/windows.rs index e6e12e18..2ab7bead 100644 --- a/src/backends/windows.rs +++ b/src/backends/windows.rs @@ -34,6 +34,7 @@ pub use crate::util::{inner_u32, inner_u64}; // https://github.com/microsoft/windows-rs/blob/0.60.0/crates/libs/targets/src/lib.rs windows_targets::link!("bcryptprimitives.dll" "system" fn ProcessPrng(pbdata: *mut u8, cbdata: usize) -> i32); +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { let result = unsafe { ProcessPrng(dest.as_mut_ptr().cast::(), dest.len()) }; // Since Windows 10, calls to the user-mode RNG are guaranteed to never diff --git a/src/backends/windows7.rs b/src/backends/windows7.rs index 5b3c86c9..8a353a9f 100644 --- a/src/backends/windows7.rs +++ b/src/backends/windows7.rs @@ -25,6 +25,7 @@ extern "system" { type BOOLEAN = u8; const TRUE: BOOLEAN = 1u8; +#[inline] pub fn fill_inner(dest: &mut [MaybeUninit]) -> Result<(), Error> { // Prevent overflow of u32 let chunk_size = usize::try_from(i32::MAX).expect("Windows does not support 16-bit targets"); diff --git a/src/util.rs b/src/util.rs index 4a55b0a5..d42c26e7 100644 --- a/src/util.rs +++ b/src/util.rs @@ -49,6 +49,7 @@ fn ptr_from_ref(r: &T) -> *const T { } /// Default implementation of `inner_u32` on top of `fill_uninit` +#[inline] pub fn inner_u32() -> Result { let mut res = MaybeUninit::::uninit(); // SAFETY: the created slice has the same size as `res` @@ -63,6 +64,7 @@ pub fn inner_u32() -> Result { } /// Default implementation of `inner_u64` on top of `fill_uninit` +#[inline] pub fn inner_u64() -> Result { let mut res = MaybeUninit::::uninit(); // SAFETY: the created slice has the same size as `res`