1 // SPDX-License-Identifier: Apache-2.0 OR MIT 1 // SPDX-License-Identifier: Apache-2.0 OR MIT
2 2
3 //! API to safely and fallibly initialize pinn 3 //! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
4 //! 4 //!
5 //! It also allows in-place initialization of 5 //! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
6 //! overflow. 6 //! overflow.
7 //! 7 //!
8 //! Most `struct`s from the [`sync`] module ne 8 //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
9 //! `struct`s from C. [Pinning][pinning] is Ru 9 //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
10 //! 10 //!
11 //! # Overview 11 //! # Overview
12 //! 12 //!
13 //! To initialize a `struct` with an in-place 13 //! To initialize a `struct` with an in-place constructor you will need two things:
14 //! - an in-place constructor, 14 //! - an in-place constructor,
15 //! - a memory location that can hold your `st 15 //! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
16 //! [`UniqueArc<T>`], [`Box<T>`] or any othe 16 //! [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]).
17 //! 17 //!
18 //! To get an in-place constructor there are g 18 //! To get an in-place constructor there are generally three options:
19 //! - directly creating an in-place constructo 19 //! - directly creating an in-place constructor using the [`pin_init!`] macro,
20 //! - a custom function/macro returning an in- 20 //! - a custom function/macro returning an in-place constructor provided by someone else,
21 //! - using the unsafe function [`pin_init_fro 21 //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
22 //! 22 //!
23 //! Aside from pinned initialization, this API 23 //! Aside from pinned initialization, this API also supports in-place construction without pinning,
24 //! the macros/types/functions are generally n 24 //! the macros/types/functions are generally named like the pinned variants without the `pin`
25 //! prefix. 25 //! prefix.
26 //! 26 //!
27 //! # Examples 27 //! # Examples
28 //! 28 //!
29 //! ## Using the [`pin_init!`] macro 29 //! ## Using the [`pin_init!`] macro
30 //! 30 //!
31 //! If you want to use [`PinInit`], then you w 31 //! If you want to use [`PinInit`], then you will have to annotate your `struct` with
32 //! `#[`[`pin_data`]`]`. It is a macro that us 32 //! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
33 //! [structurally pinned fields]. After doing 33 //! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
34 //! [`pin_init!`]. The syntax is almost the sa 34 //! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
35 //! that you need to write `<-` instead of `:` 35 //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
36 //! 36 //!
37 //! ```rust 37 //! ```rust
38 //! # #![allow(clippy::disallowed_names)] !! 38 //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
39 //! use kernel::sync::{new_mutex, Mutex}; !! 39 //! use kernel::{prelude::*, sync::Mutex, new_mutex};
40 //! # use core::pin::Pin; 40 //! # use core::pin::Pin;
41 //! #[pin_data] 41 //! #[pin_data]
42 //! struct Foo { 42 //! struct Foo {
43 //! #[pin] 43 //! #[pin]
44 //! a: Mutex<usize>, 44 //! a: Mutex<usize>,
45 //! b: u32, 45 //! b: u32,
46 //! } 46 //! }
47 //! 47 //!
48 //! let foo = pin_init!(Foo { 48 //! let foo = pin_init!(Foo {
49 //! a <- new_mutex!(42, "Foo::a"), 49 //! a <- new_mutex!(42, "Foo::a"),
50 //! b: 24, 50 //! b: 24,
51 //! }); 51 //! });
52 //! ``` 52 //! ```
53 //! 53 //!
54 //! `foo` now is of the type [`impl PinInit<Fo 54 //! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
55 //! (or just the stack) to actually initialize 55 //! (or just the stack) to actually initialize a `Foo`:
56 //! 56 //!
57 //! ```rust 57 //! ```rust
58 //! # #![allow(clippy::disallowed_names)] !! 58 //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
59 //! # use kernel::sync::{new_mutex, Mutex}; !! 59 //! # use kernel::{prelude::*, sync::Mutex, new_mutex};
60 //! # use core::pin::Pin; 60 //! # use core::pin::Pin;
61 //! # #[pin_data] 61 //! # #[pin_data]
62 //! # struct Foo { 62 //! # struct Foo {
63 //! # #[pin] 63 //! # #[pin]
64 //! # a: Mutex<usize>, 64 //! # a: Mutex<usize>,
65 //! # b: u32, 65 //! # b: u32,
66 //! # } 66 //! # }
67 //! # let foo = pin_init!(Foo { 67 //! # let foo = pin_init!(Foo {
68 //! # a <- new_mutex!(42, "Foo::a"), 68 //! # a <- new_mutex!(42, "Foo::a"),
69 //! # b: 24, 69 //! # b: 24,
70 //! # }); 70 //! # });
71 //! let foo: Result<Pin<Box<Foo>>> = Box::pin_ !! 71 //! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo);
72 //! ``` 72 //! ```
73 //! 73 //!
74 //! For more information see the [`pin_init!`] 74 //! For more information see the [`pin_init!`] macro.
75 //! 75 //!
76 //! ## Using a custom function/macro that retu 76 //! ## Using a custom function/macro that returns an initializer
77 //! 77 //!
78 //! Many types from the kernel supply a functi 78 //! Many types from the kernel supply a function/macro that returns an initializer, because the
79 //! above method only works for types where yo 79 //! above method only works for types where you can access the fields.
80 //! 80 //!
81 //! ```rust 81 //! ```rust
82 //! # use kernel::sync::{new_mutex, Arc, Mutex !! 82 //! # use kernel::{new_mutex, sync::{Arc, Mutex}};
83 //! let mtx: Result<Arc<Mutex<usize>>> = !! 83 //! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx"));
84 //! Arc::pin_init(new_mutex!(42, "example: <<
85 //! ``` 84 //! ```
86 //! 85 //!
87 //! To declare an init macro/function you just 86 //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
88 //! 87 //!
89 //! ```rust 88 //! ```rust
90 //! # #![allow(clippy::disallowed_names)] !! 89 //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
91 //! # use kernel::{sync::Mutex, new_mutex, ini !! 90 //! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init};
92 //! #[pin_data] 91 //! #[pin_data]
93 //! struct DriverData { 92 //! struct DriverData {
94 //! #[pin] 93 //! #[pin]
95 //! status: Mutex<i32>, 94 //! status: Mutex<i32>,
96 //! buffer: Box<[u8; 1_000_000]>, 95 //! buffer: Box<[u8; 1_000_000]>,
97 //! } 96 //! }
98 //! 97 //!
99 //! impl DriverData { 98 //! impl DriverData {
100 //! fn new() -> impl PinInit<Self, Error> 99 //! fn new() -> impl PinInit<Self, Error> {
101 //! try_pin_init!(Self { 100 //! try_pin_init!(Self {
102 //! status <- new_mutex!(0, "Drive 101 //! status <- new_mutex!(0, "DriverData::status"),
103 //! buffer: Box::init(kernel::init !! 102 //! buffer: Box::init(kernel::init::zeroed())?,
104 //! }) 103 //! })
105 //! } 104 //! }
106 //! } 105 //! }
107 //! ``` 106 //! ```
108 //! 107 //!
109 //! ## Manual creation of an initializer 108 //! ## Manual creation of an initializer
110 //! 109 //!
111 //! Often when working with primitives the pre 110 //! Often when working with primitives the previous approaches are not sufficient. That is where
112 //! [`pin_init_from_closure()`] comes in. This 111 //! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
113 //! [`impl PinInit<T, E>`] directly from a clo 112 //! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
114 //! actually does the initialization in the co 113 //! actually does the initialization in the correct way. Here are the things to look out for
115 //! (we are calling the parameter to the closu 114 //! (we are calling the parameter to the closure `slot`):
116 //! - when the closure returns `Ok(())`, then 115 //! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
117 //! `slot` now contains a valid bit pattern 116 //! `slot` now contains a valid bit pattern for the type `T`,
118 //! - when the closure returns `Err(e)`, then 117 //! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
119 //! you need to take care to clean up anythi 118 //! you need to take care to clean up anything if your initialization fails mid-way,
120 //! - you may assume that `slot` will stay pin 119 //! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
121 //! `slot` gets called. 120 //! `slot` gets called.
122 //! 121 //!
123 //! ```rust 122 //! ```rust
124 //! # #![allow(unreachable_pub, clippy::disall !! 123 //! use kernel::{prelude::*, init};
125 //! use kernel::{init, types::Opaque}; <<
126 //! use core::{ptr::addr_of_mut, marker::Phant 124 //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
127 //! # mod bindings { 125 //! # mod bindings {
128 //! # #![allow(non_camel_case_types)] <<
129 //! # pub struct foo; 126 //! # pub struct foo;
130 //! # pub unsafe fn init_foo(_ptr: *mut fo 127 //! # pub unsafe fn init_foo(_ptr: *mut foo) {}
131 //! # pub unsafe fn destroy_foo(_ptr: *mut 128 //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
132 //! # pub unsafe fn enable_foo(_ptr: *mut 129 //! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
133 //! # } 130 //! # }
134 //! # // `Error::from_errno` is `pub(crate)` i <<
135 //! # trait FromErrno { <<
136 //! # fn from_errno(errno: core::ffi::c_in <<
137 //! # // Dummy error that can be const <<
138 //! # Error::from(core::fmt::Error) <<
139 //! # } <<
140 //! # } <<
141 //! # impl FromErrno for Error {} <<
142 //! /// # Invariants 131 //! /// # Invariants
143 //! /// 132 //! ///
144 //! /// `foo` is always initialized 133 //! /// `foo` is always initialized
145 //! #[pin_data(PinnedDrop)] 134 //! #[pin_data(PinnedDrop)]
146 //! pub struct RawFoo { 135 //! pub struct RawFoo {
147 //! #[pin] 136 //! #[pin]
148 //! foo: Opaque<bindings::foo>, 137 //! foo: Opaque<bindings::foo>,
149 //! #[pin] 138 //! #[pin]
150 //! _p: PhantomPinned, 139 //! _p: PhantomPinned,
151 //! } 140 //! }
152 //! 141 //!
153 //! impl RawFoo { 142 //! impl RawFoo {
154 //! pub fn new(flags: u32) -> impl PinInit 143 //! pub fn new(flags: u32) -> impl PinInit<Self, Error> {
155 //! // SAFETY: 144 //! // SAFETY:
156 //! // - when the closure returns `Ok( 145 //! // - when the closure returns `Ok(())`, then it has successfully initialized and
157 //! // enabled `foo`, 146 //! // enabled `foo`,
158 //! // - when it returns `Err(e)`, the 147 //! // - when it returns `Err(e)`, then it has cleaned up before
159 //! unsafe { 148 //! unsafe {
160 //! init::pin_init_from_closure(mo 149 //! init::pin_init_from_closure(move |slot: *mut Self| {
161 //! // `slot` contains uninit 150 //! // `slot` contains uninit memory, avoid creating a reference.
162 //! let foo = addr_of_mut!((*s 151 //! let foo = addr_of_mut!((*slot).foo);
163 //! 152 //!
164 //! // Initialize the `foo` 153 //! // Initialize the `foo`
165 //! bindings::init_foo(Opaque: 154 //! bindings::init_foo(Opaque::raw_get(foo));
166 //! 155 //!
167 //! // Try to enable it. 156 //! // Try to enable it.
168 //! let err = bindings::enable 157 //! let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
169 //! if err != 0 { 158 //! if err != 0 {
170 //! // Enabling has failed 159 //! // Enabling has failed, first clean up the foo and then return the error.
171 //! bindings::destroy_foo( 160 //! bindings::destroy_foo(Opaque::raw_get(foo));
172 //! return Err(Error::from !! 161 //! return Err(Error::from_kernel_errno(err));
173 //! } 162 //! }
174 //! 163 //!
175 //! // All fields of `RawFoo` 164 //! // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
176 //! Ok(()) 165 //! Ok(())
177 //! }) 166 //! })
178 //! } 167 //! }
179 //! } 168 //! }
180 //! } 169 //! }
181 //! 170 //!
182 //! #[pinned_drop] 171 //! #[pinned_drop]
183 //! impl PinnedDrop for RawFoo { 172 //! impl PinnedDrop for RawFoo {
184 //! fn drop(self: Pin<&mut Self>) { 173 //! fn drop(self: Pin<&mut Self>) {
185 //! // SAFETY: Since `foo` is initiali 174 //! // SAFETY: Since `foo` is initialized, destroying is safe.
186 //! unsafe { bindings::destroy_foo(sel 175 //! unsafe { bindings::destroy_foo(self.foo.get()) };
187 //! } 176 //! }
188 //! } 177 //! }
189 //! ``` 178 //! ```
190 //! 179 //!
191 //! For the special case where initializing a 180 //! For the special case where initializing a field is a single FFI-function call that cannot fail,
192 //! there exist the helper function [`Opaque:: 181 //! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
193 //! [`Opaque`] field by just delegating to the 182 //! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
194 //! with [`pin_init!`]. 183 //! with [`pin_init!`].
195 //! 184 //!
196 //! For more information on how to use [`pin_i 185 //! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
197 //! the `kernel` crate. The [`sync`] module is 186 //! the `kernel` crate. The [`sync`] module is a good starting point.
198 //! 187 //!
199 //! [`sync`]: kernel::sync 188 //! [`sync`]: kernel::sync
200 //! [pinning]: https://doc.rust-lang.org/std/p 189 //! [pinning]: https://doc.rust-lang.org/std/pin/index.html
201 //! [structurally pinned fields]: 190 //! [structurally pinned fields]:
202 //! https://doc.rust-lang.org/std/pin/inde 191 //! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
203 //! [stack]: crate::stack_pin_init 192 //! [stack]: crate::stack_pin_init
204 //! [`Arc<T>`]: crate::sync::Arc 193 //! [`Arc<T>`]: crate::sync::Arc
205 //! [`impl PinInit<Foo>`]: PinInit 194 //! [`impl PinInit<Foo>`]: PinInit
206 //! [`impl PinInit<T, E>`]: PinInit 195 //! [`impl PinInit<T, E>`]: PinInit
207 //! [`impl Init<T, E>`]: Init 196 //! [`impl Init<T, E>`]: Init
208 //! [`Opaque`]: kernel::types::Opaque 197 //! [`Opaque`]: kernel::types::Opaque
209 //! [`Opaque::ffi_init`]: kernel::types::Opaqu 198 //! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
210 //! [`pin_data`]: ::macros::pin_data 199 //! [`pin_data`]: ::macros::pin_data
211 //! [`pin_init!`]: crate::pin_init! 200 //! [`pin_init!`]: crate::pin_init!
212 201
213 use crate::{ 202 use crate::{
214 alloc::{box_ext::BoxExt, AllocError, Flags <<
215 error::{self, Error}, 203 error::{self, Error},
216 sync::Arc, <<
217 sync::UniqueArc, 204 sync::UniqueArc,
218 types::{Opaque, ScopeGuard}, <<
219 }; 205 };
220 use alloc::boxed::Box; 206 use alloc::boxed::Box;
221 use core::{ 207 use core::{
222 cell::UnsafeCell, !! 208 alloc::AllocError,
>> 209 cell::Cell,
223 convert::Infallible, 210 convert::Infallible,
224 marker::PhantomData, 211 marker::PhantomData,
225 mem::MaybeUninit, 212 mem::MaybeUninit,
226 num::*, 213 num::*,
227 pin::Pin, 214 pin::Pin,
228 ptr::{self, NonNull}, 215 ptr::{self, NonNull},
229 }; 216 };
230 217
231 #[doc(hidden)] 218 #[doc(hidden)]
232 pub mod __internal; 219 pub mod __internal;
233 #[doc(hidden)] 220 #[doc(hidden)]
234 pub mod macros; 221 pub mod macros;
235 222
236 /// Initialize and pin a type directly on the 223 /// Initialize and pin a type directly on the stack.
237 /// 224 ///
238 /// # Examples 225 /// # Examples
239 /// 226 ///
240 /// ```rust 227 /// ```rust
241 /// # #![allow(clippy::disallowed_names)] !! 228 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
242 /// # use kernel::{init, macros::pin_data, pin !! 229 /// # use kernel::{init, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
>> 230 /// # use macros::pin_data;
243 /// # use core::pin::Pin; 231 /// # use core::pin::Pin;
244 /// #[pin_data] 232 /// #[pin_data]
245 /// struct Foo { 233 /// struct Foo {
246 /// #[pin] 234 /// #[pin]
247 /// a: Mutex<usize>, 235 /// a: Mutex<usize>,
248 /// b: Bar, 236 /// b: Bar,
249 /// } 237 /// }
250 /// 238 ///
251 /// #[pin_data] 239 /// #[pin_data]
252 /// struct Bar { 240 /// struct Bar {
253 /// x: u32, 241 /// x: u32,
254 /// } 242 /// }
255 /// 243 ///
256 /// stack_pin_init!(let foo = pin_init!(Foo { 244 /// stack_pin_init!(let foo = pin_init!(Foo {
257 /// a <- new_mutex!(42), 245 /// a <- new_mutex!(42),
258 /// b: Bar { 246 /// b: Bar {
259 /// x: 64, 247 /// x: 64,
260 /// }, 248 /// },
261 /// })); 249 /// }));
262 /// let foo: Pin<&mut Foo> = foo; 250 /// let foo: Pin<&mut Foo> = foo;
263 /// pr_info!("a: {}", &*foo.a.lock()); 251 /// pr_info!("a: {}", &*foo.a.lock());
264 /// ``` 252 /// ```
265 /// 253 ///
266 /// # Syntax 254 /// # Syntax
267 /// 255 ///
268 /// A normal `let` binding with optional type 256 /// A normal `let` binding with optional type annotation. The expression is expected to implement
269 /// [`PinInit`]/[`Init`] with the error type [ 257 /// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
270 /// type, then use [`stack_try_pin_init!`]. 258 /// type, then use [`stack_try_pin_init!`].
271 /// 259 ///
272 /// [`stack_try_pin_init!`]: crate::stack_try_ 260 /// [`stack_try_pin_init!`]: crate::stack_try_pin_init!
273 #[macro_export] 261 #[macro_export]
274 macro_rules! stack_pin_init { 262 macro_rules! stack_pin_init {
275 (let $var:ident $(: $t:ty)? = $val:expr) = 263 (let $var:ident $(: $t:ty)? = $val:expr) => {
276 let val = $val; 264 let val = $val;
277 let mut $var = ::core::pin::pin!($crat 265 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
278 let mut $var = match $crate::init::__i 266 let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
279 Ok(res) => res, 267 Ok(res) => res,
280 Err(x) => { 268 Err(x) => {
281 let x: ::core::convert::Infall 269 let x: ::core::convert::Infallible = x;
282 match x {} 270 match x {}
283 } 271 }
284 }; 272 };
285 }; 273 };
286 } 274 }
287 275
288 /// Initialize and pin a type directly on the 276 /// Initialize and pin a type directly on the stack.
289 /// 277 ///
290 /// # Examples 278 /// # Examples
291 /// 279 ///
292 /// ```rust,ignore !! 280 /// ```rust
293 /// # #![allow(clippy::disallowed_names)] !! 281 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
294 /// # use kernel::{init, pin_init, stack_try_p 282 /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
295 /// # use macros::pin_data; 283 /// # use macros::pin_data;
296 /// # use core::{alloc::AllocError, pin::Pin}; 284 /// # use core::{alloc::AllocError, pin::Pin};
297 /// #[pin_data] 285 /// #[pin_data]
298 /// struct Foo { 286 /// struct Foo {
299 /// #[pin] 287 /// #[pin]
300 /// a: Mutex<usize>, 288 /// a: Mutex<usize>,
301 /// b: Box<Bar>, 289 /// b: Box<Bar>,
302 /// } 290 /// }
303 /// 291 ///
304 /// struct Bar { 292 /// struct Bar {
305 /// x: u32, 293 /// x: u32,
306 /// } 294 /// }
307 /// 295 ///
308 /// stack_try_pin_init!(let foo: Result<Pin<&m 296 /// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
309 /// a <- new_mutex!(42), 297 /// a <- new_mutex!(42),
310 /// b: Box::new(Bar { !! 298 /// b: Box::try_new(Bar {
311 /// x: 64, 299 /// x: 64,
312 /// }, GFP_KERNEL)?, !! 300 /// })?,
313 /// })); 301 /// }));
314 /// let foo = foo.unwrap(); 302 /// let foo = foo.unwrap();
315 /// pr_info!("a: {}", &*foo.a.lock()); 303 /// pr_info!("a: {}", &*foo.a.lock());
316 /// ``` 304 /// ```
317 /// 305 ///
318 /// ```rust,ignore !! 306 /// ```rust
319 /// # #![allow(clippy::disallowed_names)] !! 307 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
320 /// # use kernel::{init, pin_init, stack_try_p 308 /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex};
321 /// # use macros::pin_data; 309 /// # use macros::pin_data;
322 /// # use core::{alloc::AllocError, pin::Pin}; 310 /// # use core::{alloc::AllocError, pin::Pin};
323 /// #[pin_data] 311 /// #[pin_data]
324 /// struct Foo { 312 /// struct Foo {
325 /// #[pin] 313 /// #[pin]
326 /// a: Mutex<usize>, 314 /// a: Mutex<usize>,
327 /// b: Box<Bar>, 315 /// b: Box<Bar>,
328 /// } 316 /// }
329 /// 317 ///
330 /// struct Bar { 318 /// struct Bar {
331 /// x: u32, 319 /// x: u32,
332 /// } 320 /// }
333 /// 321 ///
334 /// stack_try_pin_init!(let foo: Pin<&mut Foo> 322 /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
335 /// a <- new_mutex!(42), 323 /// a <- new_mutex!(42),
336 /// b: Box::new(Bar { !! 324 /// b: Box::try_new(Bar {
337 /// x: 64, 325 /// x: 64,
338 /// }, GFP_KERNEL)?, !! 326 /// })?,
339 /// })); 327 /// }));
340 /// pr_info!("a: {}", &*foo.a.lock()); 328 /// pr_info!("a: {}", &*foo.a.lock());
341 /// # Ok::<_, AllocError>(()) 329 /// # Ok::<_, AllocError>(())
342 /// ``` 330 /// ```
343 /// 331 ///
344 /// # Syntax 332 /// # Syntax
345 /// 333 ///
346 /// A normal `let` binding with optional type 334 /// A normal `let` binding with optional type annotation. The expression is expected to implement
347 /// [`PinInit`]/[`Init`]. This macro assigns a 335 /// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
348 /// `=` will propagate this error. 336 /// `=` will propagate this error.
349 #[macro_export] 337 #[macro_export]
350 macro_rules! stack_try_pin_init { 338 macro_rules! stack_try_pin_init {
351 (let $var:ident $(: $t:ty)? = $val:expr) = 339 (let $var:ident $(: $t:ty)? = $val:expr) => {
352 let val = $val; 340 let val = $val;
353 let mut $var = ::core::pin::pin!($crat 341 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
354 let mut $var = $crate::init::__interna 342 let mut $var = $crate::init::__internal::StackInit::init($var, val);
355 }; 343 };
356 (let $var:ident $(: $t:ty)? =? $val:expr) 344 (let $var:ident $(: $t:ty)? =? $val:expr) => {
357 let val = $val; 345 let val = $val;
358 let mut $var = ::core::pin::pin!($crat 346 let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
359 let mut $var = $crate::init::__interna 347 let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
360 }; 348 };
361 } 349 }
362 350
363 /// Construct an in-place, pinned initializer 351 /// Construct an in-place, pinned initializer for `struct`s.
364 /// 352 ///
365 /// This macro defaults the error to [`Infalli 353 /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
366 /// [`try_pin_init!`]. 354 /// [`try_pin_init!`].
367 /// 355 ///
368 /// The syntax is almost identical to that of 356 /// The syntax is almost identical to that of a normal `struct` initializer:
369 /// 357 ///
370 /// ```rust 358 /// ```rust
371 /// # #![allow(clippy::disallowed_names)] !! 359 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
372 /// # use kernel::{init, pin_init, macros::pin 360 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
373 /// # use core::pin::Pin; 361 /// # use core::pin::Pin;
374 /// #[pin_data] 362 /// #[pin_data]
375 /// struct Foo { 363 /// struct Foo {
376 /// a: usize, 364 /// a: usize,
377 /// b: Bar, 365 /// b: Bar,
378 /// } 366 /// }
379 /// 367 ///
380 /// #[pin_data] 368 /// #[pin_data]
381 /// struct Bar { 369 /// struct Bar {
382 /// x: u32, 370 /// x: u32,
383 /// } 371 /// }
384 /// 372 ///
385 /// # fn demo() -> impl PinInit<Foo> { 373 /// # fn demo() -> impl PinInit<Foo> {
386 /// let a = 42; 374 /// let a = 42;
387 /// 375 ///
388 /// let initializer = pin_init!(Foo { 376 /// let initializer = pin_init!(Foo {
389 /// a, 377 /// a,
390 /// b: Bar { 378 /// b: Bar {
391 /// x: 64, 379 /// x: 64,
392 /// }, 380 /// },
393 /// }); 381 /// });
394 /// # initializer } 382 /// # initializer }
395 /// # Box::pin_init(demo(), GFP_KERNEL).unwrap !! 383 /// # Box::pin_init(demo()).unwrap();
396 /// ``` 384 /// ```
397 /// 385 ///
398 /// Arbitrary Rust expressions can be used to 386 /// Arbitrary Rust expressions can be used to set the value of a variable.
399 /// 387 ///
400 /// The fields are initialized in the order th 388 /// The fields are initialized in the order that they appear in the initializer. So it is possible
401 /// to read already initialized fields using r 389 /// to read already initialized fields using raw pointers.
402 /// 390 ///
403 /// IMPORTANT: You are not allowed to create r 391 /// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
404 /// initializer. 392 /// initializer.
405 /// 393 ///
406 /// # Init-functions 394 /// # Init-functions
407 /// 395 ///
408 /// When working with this API it is often des 396 /// When working with this API it is often desired to let others construct your types without
409 /// giving access to all fields. This is where 397 /// giving access to all fields. This is where you would normally write a plain function `new`
410 /// that would return a new instance of your t 398 /// that would return a new instance of your type. With this API that is also possible.
411 /// However, there are a few extra things to k 399 /// However, there are a few extra things to keep in mind.
412 /// 400 ///
413 /// To create an initializer function, simply 401 /// To create an initializer function, simply declare it like this:
414 /// 402 ///
415 /// ```rust 403 /// ```rust
416 /// # #![allow(clippy::disallowed_names)] !! 404 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
417 /// # use kernel::{init, pin_init, init::*}; !! 405 /// # use kernel::{init, pin_init, prelude::*, init::*};
418 /// # use core::pin::Pin; 406 /// # use core::pin::Pin;
419 /// # #[pin_data] 407 /// # #[pin_data]
420 /// # struct Foo { 408 /// # struct Foo {
421 /// # a: usize, 409 /// # a: usize,
422 /// # b: Bar, 410 /// # b: Bar,
423 /// # } 411 /// # }
424 /// # #[pin_data] 412 /// # #[pin_data]
425 /// # struct Bar { 413 /// # struct Bar {
426 /// # x: u32, 414 /// # x: u32,
427 /// # } 415 /// # }
428 /// impl Foo { 416 /// impl Foo {
429 /// fn new() -> impl PinInit<Self> { 417 /// fn new() -> impl PinInit<Self> {
430 /// pin_init!(Self { 418 /// pin_init!(Self {
431 /// a: 42, 419 /// a: 42,
432 /// b: Bar { 420 /// b: Bar {
433 /// x: 64, 421 /// x: 64,
434 /// }, 422 /// },
435 /// }) 423 /// })
436 /// } 424 /// }
437 /// } 425 /// }
438 /// ``` 426 /// ```
439 /// 427 ///
440 /// Users of `Foo` can now create it like this 428 /// Users of `Foo` can now create it like this:
441 /// 429 ///
442 /// ```rust 430 /// ```rust
443 /// # #![allow(clippy::disallowed_names)] !! 431 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
444 /// # use kernel::{init, pin_init, macros::pin 432 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
445 /// # use core::pin::Pin; 433 /// # use core::pin::Pin;
446 /// # #[pin_data] 434 /// # #[pin_data]
447 /// # struct Foo { 435 /// # struct Foo {
448 /// # a: usize, 436 /// # a: usize,
449 /// # b: Bar, 437 /// # b: Bar,
450 /// # } 438 /// # }
451 /// # #[pin_data] 439 /// # #[pin_data]
452 /// # struct Bar { 440 /// # struct Bar {
453 /// # x: u32, 441 /// # x: u32,
454 /// # } 442 /// # }
455 /// # impl Foo { 443 /// # impl Foo {
456 /// # fn new() -> impl PinInit<Self> { 444 /// # fn new() -> impl PinInit<Self> {
457 /// # pin_init!(Self { 445 /// # pin_init!(Self {
458 /// # a: 42, 446 /// # a: 42,
459 /// # b: Bar { 447 /// # b: Bar {
460 /// # x: 64, 448 /// # x: 64,
461 /// # }, 449 /// # },
462 /// # }) 450 /// # })
463 /// # } 451 /// # }
464 /// # } 452 /// # }
465 /// let foo = Box::pin_init(Foo::new(), GFP_KE !! 453 /// let foo = Box::pin_init(Foo::new());
466 /// ``` 454 /// ```
467 /// 455 ///
468 /// They can also easily embed it into their o 456 /// They can also easily embed it into their own `struct`s:
469 /// 457 ///
470 /// ```rust 458 /// ```rust
471 /// # #![allow(clippy::disallowed_names)] !! 459 /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)]
472 /// # use kernel::{init, pin_init, macros::pin 460 /// # use kernel::{init, pin_init, macros::pin_data, init::*};
473 /// # use core::pin::Pin; 461 /// # use core::pin::Pin;
474 /// # #[pin_data] 462 /// # #[pin_data]
475 /// # struct Foo { 463 /// # struct Foo {
476 /// # a: usize, 464 /// # a: usize,
477 /// # b: Bar, 465 /// # b: Bar,
478 /// # } 466 /// # }
479 /// # #[pin_data] 467 /// # #[pin_data]
480 /// # struct Bar { 468 /// # struct Bar {
481 /// # x: u32, 469 /// # x: u32,
482 /// # } 470 /// # }
483 /// # impl Foo { 471 /// # impl Foo {
484 /// # fn new() -> impl PinInit<Self> { 472 /// # fn new() -> impl PinInit<Self> {
485 /// # pin_init!(Self { 473 /// # pin_init!(Self {
486 /// # a: 42, 474 /// # a: 42,
487 /// # b: Bar { 475 /// # b: Bar {
488 /// # x: 64, 476 /// # x: 64,
489 /// # }, 477 /// # },
490 /// # }) 478 /// # })
491 /// # } 479 /// # }
492 /// # } 480 /// # }
493 /// #[pin_data] 481 /// #[pin_data]
494 /// struct FooContainer { 482 /// struct FooContainer {
495 /// #[pin] 483 /// #[pin]
496 /// foo1: Foo, 484 /// foo1: Foo,
497 /// #[pin] 485 /// #[pin]
498 /// foo2: Foo, 486 /// foo2: Foo,
499 /// other: u32, 487 /// other: u32,
500 /// } 488 /// }
501 /// 489 ///
502 /// impl FooContainer { 490 /// impl FooContainer {
503 /// fn new(other: u32) -> impl PinInit<Sel 491 /// fn new(other: u32) -> impl PinInit<Self> {
504 /// pin_init!(Self { 492 /// pin_init!(Self {
505 /// foo1 <- Foo::new(), 493 /// foo1 <- Foo::new(),
506 /// foo2 <- Foo::new(), 494 /// foo2 <- Foo::new(),
507 /// other, 495 /// other,
508 /// }) 496 /// })
509 /// } 497 /// }
510 /// } 498 /// }
511 /// ``` 499 /// ```
512 /// 500 ///
513 /// Here we see that when using `pin_init!` wi 501 /// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
514 /// This signifies that the given field is ini 502 /// This signifies that the given field is initialized in-place. As with `struct` initializers, just
515 /// writing the field (in this case `other`) w 503 /// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
516 /// 504 ///
517 /// # Syntax 505 /// # Syntax
518 /// 506 ///
519 /// As already mentioned in the examples above 507 /// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
520 /// the following modifications is expected: 508 /// the following modifications is expected:
521 /// - Fields that you want to initialize in-pl 509 /// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
522 /// - In front of the initializer you can writ 510 /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
523 /// pointer named `this` inside of the initi 511 /// pointer named `this` inside of the initializer.
524 /// - Using struct update syntax one can place <<
525 /// struct, this initializes every field wit <<
526 /// body. This can only be done if [`Zeroabl <<
527 /// 512 ///
528 /// For instance: 513 /// For instance:
529 /// 514 ///
530 /// ```rust 515 /// ```rust
531 /// # use kernel::{macros::{Zeroable, pin_data !! 516 /// # use kernel::pin_init;
>> 517 /// # use macros::pin_data;
532 /// # use core::{ptr::addr_of_mut, marker::Pha 518 /// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
533 /// #[pin_data] 519 /// #[pin_data]
534 /// #[derive(Zeroable)] <<
535 /// struct Buf { 520 /// struct Buf {
536 /// // `ptr` points into `buf`. 521 /// // `ptr` points into `buf`.
537 /// ptr: *mut u8, 522 /// ptr: *mut u8,
538 /// buf: [u8; 64], 523 /// buf: [u8; 64],
539 /// #[pin] 524 /// #[pin]
540 /// pin: PhantomPinned, 525 /// pin: PhantomPinned,
541 /// } 526 /// }
542 /// pin_init!(&this in Buf { 527 /// pin_init!(&this in Buf {
543 /// buf: [0; 64], 528 /// buf: [0; 64],
544 /// ptr: unsafe { addr_of_mut!((*this.as_p 529 /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
545 /// pin: PhantomPinned, 530 /// pin: PhantomPinned,
546 /// }); 531 /// });
547 /// pin_init!(Buf { <<
548 /// buf: [1; 64], <<
549 /// ..Zeroable::zeroed() <<
550 /// }); <<
551 /// ``` 532 /// ```
552 /// 533 ///
553 /// [`try_pin_init!`]: kernel::try_pin_init 534 /// [`try_pin_init!`]: kernel::try_pin_init
554 /// [`NonNull<Self>`]: core::ptr::NonNull 535 /// [`NonNull<Self>`]: core::ptr::NonNull
555 // For a detailed example of how this macro wo 536 // For a detailed example of how this macro works, see the module documentation of the hidden
556 // module `__internal` inside of `init/__inter 537 // module `__internal` inside of `init/__internal.rs`.
557 #[macro_export] 538 #[macro_export]
558 macro_rules! pin_init { 539 macro_rules! pin_init {
559 ($(&$this:ident in)? $t:ident $(::<$($gene 540 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
560 $($fields:tt)* 541 $($fields:tt)*
561 }) => { 542 }) => {
562 $crate::__init_internal!( !! 543 $crate::try_pin_init!(
563 @this($($this)?), 544 @this($($this)?),
564 @typ($t $(::<$($generics),*>)?), 545 @typ($t $(::<$($generics),*>)?),
565 @fields($($fields)*), 546 @fields($($fields)*),
566 @error(::core::convert::Infallible 547 @error(::core::convert::Infallible),
567 @data(PinData, use_data), <<
568 @has_data(HasPinData, __pin_data), <<
569 @construct_closure(pin_init_from_c <<
570 @munch_fields($($fields)*), <<
571 ) 548 )
572 }; 549 };
573 } 550 }
574 551
575 /// Construct an in-place, fallible pinned ini 552 /// Construct an in-place, fallible pinned initializer for `struct`s.
576 /// 553 ///
577 /// If the initialization can complete without 554 /// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
578 /// 555 ///
579 /// You can use the `?` operator or use `retur 556 /// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
580 /// initialization and return the error. 557 /// initialization and return the error.
581 /// 558 ///
582 /// IMPORTANT: if you have `unsafe` code insid 559 /// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
583 /// initialization fails, the memory can be sa 560 /// initialization fails, the memory can be safely deallocated without any further modifications.
584 /// 561 ///
585 /// This macro defaults the error to [`Error`] 562 /// This macro defaults the error to [`Error`].
586 /// 563 ///
587 /// The syntax is identical to [`pin_init!`] w 564 /// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
588 /// after the `struct` initializer to specify 565 /// after the `struct` initializer to specify the error type you want to use.
589 /// 566 ///
590 /// # Examples 567 /// # Examples
591 /// 568 ///
592 /// ```rust 569 /// ```rust
593 /// # #![feature(new_uninit)] 570 /// # #![feature(new_uninit)]
594 /// use kernel::{init::{self, PinInit}, error: 571 /// use kernel::{init::{self, PinInit}, error::Error};
595 /// #[pin_data] 572 /// #[pin_data]
596 /// struct BigBuf { 573 /// struct BigBuf {
597 /// big: Box<[u8; 1024 * 1024 * 1024]>, 574 /// big: Box<[u8; 1024 * 1024 * 1024]>,
598 /// small: [u8; 1024 * 1024], 575 /// small: [u8; 1024 * 1024],
599 /// ptr: *mut u8, 576 /// ptr: *mut u8,
600 /// } 577 /// }
601 /// 578 ///
602 /// impl BigBuf { 579 /// impl BigBuf {
603 /// fn new() -> impl PinInit<Self, Error> 580 /// fn new() -> impl PinInit<Self, Error> {
604 /// try_pin_init!(Self { 581 /// try_pin_init!(Self {
605 /// big: Box::init(init::zeroed(), !! 582 /// big: Box::init(init::zeroed())?,
606 /// small: [0; 1024 * 1024], 583 /// small: [0; 1024 * 1024],
607 /// ptr: core::ptr::null_mut(), 584 /// ptr: core::ptr::null_mut(),
608 /// }? Error) 585 /// }? Error)
609 /// } 586 /// }
610 /// } 587 /// }
611 /// ``` 588 /// ```
612 // For a detailed example of how this macro wo 589 // For a detailed example of how this macro works, see the module documentation of the hidden
613 // module `__internal` inside of `init/__inter 590 // module `__internal` inside of `init/__internal.rs`.
614 #[macro_export] 591 #[macro_export]
615 macro_rules! try_pin_init { 592 macro_rules! try_pin_init {
616 ($(&$this:ident in)? $t:ident $(::<$($gene 593 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
617 $($fields:tt)* 594 $($fields:tt)*
618 }) => { 595 }) => {
619 $crate::__init_internal!( !! 596 $crate::try_pin_init!(
620 @this($($this)?), 597 @this($($this)?),
621 @typ($t $(::<$($generics),*>)? ), 598 @typ($t $(::<$($generics),*>)? ),
622 @fields($($fields)*), 599 @fields($($fields)*),
623 @error($crate::error::Error), 600 @error($crate::error::Error),
624 @data(PinData, use_data), <<
625 @has_data(HasPinData, __pin_data), <<
626 @construct_closure(pin_init_from_c <<
627 @munch_fields($($fields)*), <<
628 ) 601 )
629 }; 602 };
630 ($(&$this:ident in)? $t:ident $(::<$($gene 603 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
631 $($fields:tt)* 604 $($fields:tt)*
632 }? $err:ty) => { 605 }? $err:ty) => {
633 $crate::__init_internal!( !! 606 $crate::try_pin_init!(
634 @this($($this)?), 607 @this($($this)?),
635 @typ($t $(::<$($generics),*>)? ), 608 @typ($t $(::<$($generics),*>)? ),
636 @fields($($fields)*), 609 @fields($($fields)*),
637 @error($err), 610 @error($err),
638 @data(PinData, use_data), <<
639 @has_data(HasPinData, __pin_data), <<
640 @construct_closure(pin_init_from_c <<
641 @munch_fields($($fields)*), <<
642 ) 611 )
643 }; 612 };
>> 613 (
>> 614 @this($($this:ident)?),
>> 615 @typ($t:ident $(::<$($generics:ty),*>)?),
>> 616 @fields($($fields:tt)*),
>> 617 @error($err:ty),
>> 618 ) => {{
>> 619 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
>> 620 // type and shadow it later when we insert the arbitrary user code. That way there will be
>> 621 // no possibility of returning without `unsafe`.
>> 622 struct __InitOk;
>> 623 // Get the pin data from the supplied type.
>> 624 let data = unsafe {
>> 625 use $crate::init::__internal::HasPinData;
>> 626 $t$(::<$($generics),*>)?::__pin_data()
>> 627 };
>> 628 // Ensure that `data` really is of type `PinData` and help with type inference:
>> 629 let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>(
>> 630 data,
>> 631 move |slot| {
>> 632 {
>> 633 // Shadow the structure so it cannot be used to return early.
>> 634 struct __InitOk;
>> 635 // Create the `this` so it can be referenced by the user inside of the
>> 636 // expressions creating the individual fields.
>> 637 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
>> 638 // Initialize every field.
>> 639 $crate::try_pin_init!(init_slot:
>> 640 @data(data),
>> 641 @slot(slot),
>> 642 @munch_fields($($fields)*,),
>> 643 );
>> 644 // We use unreachable code to ensure that all fields have been mentioned exactly
>> 645 // once, this struct initializer will still be type-checked and complain with a
>> 646 // very natural error message if a field is forgotten/mentioned more than once.
>> 647 #[allow(unreachable_code, clippy::diverging_sub_expression)]
>> 648 if false {
>> 649 $crate::try_pin_init!(make_initializer:
>> 650 @slot(slot),
>> 651 @type_name($t),
>> 652 @munch_fields($($fields)*,),
>> 653 @acc(),
>> 654 );
>> 655 }
>> 656 // Forget all guards, since initialization was a success.
>> 657 $crate::try_pin_init!(forget_guards:
>> 658 @munch_fields($($fields)*,),
>> 659 );
>> 660 }
>> 661 Ok(__InitOk)
>> 662 }
>> 663 );
>> 664 let init = move |slot| -> ::core::result::Result<(), $err> {
>> 665 init(slot).map(|__InitOk| ())
>> 666 };
>> 667 let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) };
>> 668 init
>> 669 }};
>> 670 (init_slot:
>> 671 @data($data:ident),
>> 672 @slot($slot:ident),
>> 673 @munch_fields($(,)?),
>> 674 ) => {
>> 675 // Endpoint of munching, no fields are left.
>> 676 };
>> 677 (init_slot:
>> 678 @data($data:ident),
>> 679 @slot($slot:ident),
>> 680 // In-place initialization syntax.
>> 681 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
>> 682 ) => {
>> 683 let $field = $val;
>> 684 // Call the initializer.
>> 685 //
>> 686 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
>> 687 // return when an error/panic occurs.
>> 688 // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
>> 689 unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? };
>> 690 // Create the drop guard.
>> 691 //
>> 692 // We only give access to `&DropGuard`, so it cannot be forgotten via safe code.
>> 693 //
>> 694 // SAFETY: We forget the guard later when initialization has succeeded.
>> 695 let $field = &unsafe {
>> 696 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
>> 697 };
>> 698
>> 699 $crate::try_pin_init!(init_slot:
>> 700 @data($data),
>> 701 @slot($slot),
>> 702 @munch_fields($($rest)*),
>> 703 );
>> 704 };
>> 705 (init_slot:
>> 706 @data($data:ident),
>> 707 @slot($slot:ident),
>> 708 // Direct value init, this is safe for every field.
>> 709 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
>> 710 ) => {
>> 711 $(let $field = $val;)?
>> 712 // Initialize the field.
>> 713 //
>> 714 // SAFETY: The memory at `slot` is uninitialized.
>> 715 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
>> 716 // Create the drop guard:
>> 717 //
>> 718 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
>> 719 //
>> 720 // SAFETY: We forget the guard later when initialization has succeeded.
>> 721 let $field = &unsafe {
>> 722 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
>> 723 };
>> 724
>> 725 $crate::try_pin_init!(init_slot:
>> 726 @data($data),
>> 727 @slot($slot),
>> 728 @munch_fields($($rest)*),
>> 729 );
>> 730 };
>> 731 (make_initializer:
>> 732 @slot($slot:ident),
>> 733 @type_name($t:ident),
>> 734 @munch_fields($(,)?),
>> 735 @acc($($acc:tt)*),
>> 736 ) => {
>> 737 // Endpoint, nothing more to munch, create the initializer.
>> 738 // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to
>> 739 // get the correct type inference here:
>> 740 unsafe {
>> 741 ::core::ptr::write($slot, $t {
>> 742 $($acc)*
>> 743 });
>> 744 }
>> 745 };
>> 746 (make_initializer:
>> 747 @slot($slot:ident),
>> 748 @type_name($t:ident),
>> 749 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
>> 750 @acc($($acc:tt)*),
>> 751 ) => {
>> 752 $crate::try_pin_init!(make_initializer:
>> 753 @slot($slot),
>> 754 @type_name($t),
>> 755 @munch_fields($($rest)*),
>> 756 @acc($($acc)* $field: ::core::panic!(),),
>> 757 );
>> 758 };
>> 759 (make_initializer:
>> 760 @slot($slot:ident),
>> 761 @type_name($t:ident),
>> 762 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
>> 763 @acc($($acc:tt)*),
>> 764 ) => {
>> 765 $crate::try_pin_init!(make_initializer:
>> 766 @slot($slot),
>> 767 @type_name($t),
>> 768 @munch_fields($($rest)*),
>> 769 @acc($($acc)* $field: ::core::panic!(),),
>> 770 );
>> 771 };
>> 772 (forget_guards:
>> 773 @munch_fields($(,)?),
>> 774 ) => {
>> 775 // Munching finished.
>> 776 };
>> 777 (forget_guards:
>> 778 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
>> 779 ) => {
>> 780 unsafe { $crate::init::__internal::DropGuard::forget($field) };
>> 781
>> 782 $crate::try_pin_init!(forget_guards:
>> 783 @munch_fields($($rest)*),
>> 784 );
>> 785 };
>> 786 (forget_guards:
>> 787 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
>> 788 ) => {
>> 789 unsafe { $crate::init::__internal::DropGuard::forget($field) };
>> 790
>> 791 $crate::try_pin_init!(forget_guards:
>> 792 @munch_fields($($rest)*),
>> 793 );
>> 794 };
644 } 795 }
645 796
646 /// Construct an in-place initializer for `str 797 /// Construct an in-place initializer for `struct`s.
647 /// 798 ///
648 /// This macro defaults the error to [`Infalli 799 /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
649 /// [`try_init!`]. 800 /// [`try_init!`].
650 /// 801 ///
651 /// The syntax is identical to [`pin_init!`] a 802 /// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
652 /// - `unsafe` code must guarantee either full 803 /// - `unsafe` code must guarantee either full initialization or return an error and allow
653 /// deallocation of the memory. 804 /// deallocation of the memory.
654 /// - the fields are initialized in the order 805 /// - the fields are initialized in the order given in the initializer.
655 /// - no references to fields are allowed to b 806 /// - no references to fields are allowed to be created inside of the initializer.
656 /// 807 ///
657 /// This initializer is for initializing data 808 /// This initializer is for initializing data in-place that might later be moved. If you want to
658 /// pin-initialize, use [`pin_init!`]. 809 /// pin-initialize, use [`pin_init!`].
659 /// 810 ///
660 /// [`try_init!`]: crate::try_init! 811 /// [`try_init!`]: crate::try_init!
661 // For a detailed example of how this macro wo 812 // For a detailed example of how this macro works, see the module documentation of the hidden
662 // module `__internal` inside of `init/__inter 813 // module `__internal` inside of `init/__internal.rs`.
663 #[macro_export] 814 #[macro_export]
664 macro_rules! init { 815 macro_rules! init {
665 ($(&$this:ident in)? $t:ident $(::<$($gene 816 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
666 $($fields:tt)* 817 $($fields:tt)*
667 }) => { 818 }) => {
668 $crate::__init_internal!( !! 819 $crate::try_init!(
669 @this($($this)?), 820 @this($($this)?),
670 @typ($t $(::<$($generics),*>)?), 821 @typ($t $(::<$($generics),*>)?),
671 @fields($($fields)*), 822 @fields($($fields)*),
672 @error(::core::convert::Infallible 823 @error(::core::convert::Infallible),
673 @data(InitData, /*no use_data*/), <<
674 @has_data(HasInitData, __init_data <<
675 @construct_closure(init_from_closu <<
676 @munch_fields($($fields)*), <<
677 ) 824 )
678 } 825 }
679 } 826 }
680 827
681 /// Construct an in-place fallible initializer 828 /// Construct an in-place fallible initializer for `struct`s.
682 /// 829 ///
683 /// This macro defaults the error to [`Error`] 830 /// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
684 /// [`init!`]. 831 /// [`init!`].
685 /// 832 ///
686 /// The syntax is identical to [`try_pin_init! 833 /// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
687 /// append `? $type` after the `struct` initia 834 /// append `? $type` after the `struct` initializer.
688 /// The safety caveats from [`try_pin_init!`] 835 /// The safety caveats from [`try_pin_init!`] also apply:
689 /// - `unsafe` code must guarantee either full 836 /// - `unsafe` code must guarantee either full initialization or return an error and allow
690 /// deallocation of the memory. 837 /// deallocation of the memory.
691 /// - the fields are initialized in the order 838 /// - the fields are initialized in the order given in the initializer.
692 /// - no references to fields are allowed to b 839 /// - no references to fields are allowed to be created inside of the initializer.
693 /// 840 ///
694 /// # Examples 841 /// # Examples
695 /// 842 ///
696 /// ```rust 843 /// ```rust
697 /// use kernel::{init::{PinInit, zeroed}, erro !! 844 /// use kernel::{init::PinInit, error::Error, InPlaceInit};
698 /// struct BigBuf { 845 /// struct BigBuf {
699 /// big: Box<[u8; 1024 * 1024 * 1024]>, 846 /// big: Box<[u8; 1024 * 1024 * 1024]>,
700 /// small: [u8; 1024 * 1024], 847 /// small: [u8; 1024 * 1024],
701 /// } 848 /// }
702 /// 849 ///
703 /// impl BigBuf { 850 /// impl BigBuf {
704 /// fn new() -> impl Init<Self, Error> { 851 /// fn new() -> impl Init<Self, Error> {
705 /// try_init!(Self { 852 /// try_init!(Self {
706 /// big: Box::init(zeroed(), GFP_K !! 853 /// big: Box::init(zeroed())?,
707 /// small: [0; 1024 * 1024], 854 /// small: [0; 1024 * 1024],
708 /// }? Error) 855 /// }? Error)
709 /// } 856 /// }
710 /// } 857 /// }
711 /// ``` 858 /// ```
712 // For a detailed example of how this macro wo 859 // For a detailed example of how this macro works, see the module documentation of the hidden
713 // module `__internal` inside of `init/__inter 860 // module `__internal` inside of `init/__internal.rs`.
714 #[macro_export] 861 #[macro_export]
715 macro_rules! try_init { 862 macro_rules! try_init {
716 ($(&$this:ident in)? $t:ident $(::<$($gene 863 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
717 $($fields:tt)* 864 $($fields:tt)*
718 }) => { 865 }) => {
719 $crate::__init_internal!( !! 866 $crate::try_init!(
720 @this($($this)?), 867 @this($($this)?),
721 @typ($t $(::<$($generics),*>)?), 868 @typ($t $(::<$($generics),*>)?),
722 @fields($($fields)*), 869 @fields($($fields)*),
723 @error($crate::error::Error), 870 @error($crate::error::Error),
724 @data(InitData, /*no use_data*/), <<
725 @has_data(HasInitData, __init_data <<
726 @construct_closure(init_from_closu <<
727 @munch_fields($($fields)*), <<
728 ) 871 )
729 }; 872 };
730 ($(&$this:ident in)? $t:ident $(::<$($gene 873 ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
731 $($fields:tt)* 874 $($fields:tt)*
732 }? $err:ty) => { 875 }? $err:ty) => {
733 $crate::__init_internal!( !! 876 $crate::try_init!(
734 @this($($this)?), 877 @this($($this)?),
735 @typ($t $(::<$($generics),*>)?), 878 @typ($t $(::<$($generics),*>)?),
736 @fields($($fields)*), 879 @fields($($fields)*),
737 @error($err), 880 @error($err),
738 @data(InitData, /*no use_data*/), <<
739 @has_data(HasInitData, __init_data <<
740 @construct_closure(init_from_closu <<
741 @munch_fields($($fields)*), <<
742 ) 881 )
743 }; 882 };
744 } !! 883 (
745 !! 884 @this($($this:ident)?),
746 /// Asserts that a field on a struct using `#[ !! 885 @typ($t:ident $(::<$($generics:ty),*>)?),
747 /// structurally pinned. !! 886 @fields($($fields:tt)*),
748 /// !! 887 @error($err:ty),
749 /// # Example !! 888 ) => {{
750 /// !! 889 // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
751 /// This will succeed: !! 890 // type and shadow it later when we insert the arbitrary user code. That way there will be
752 /// ``` !! 891 // no possibility of returning without `unsafe`.
753 /// use kernel::assert_pinned; !! 892 struct __InitOk;
754 /// #[pin_data] !! 893 // Get the init data from the supplied type.
755 /// struct MyStruct { !! 894 let data = unsafe {
756 /// #[pin] !! 895 use $crate::init::__internal::HasInitData;
757 /// some_field: u64, !! 896 $t$(::<$($generics),*>)?::__init_data()
758 /// } <<
759 /// <<
760 /// assert_pinned!(MyStruct, some_field, u64); <<
761 /// ``` <<
762 /// <<
763 /// This will fail: <<
764 // TODO: replace with `compile_fail` when supp <<
765 /// ```ignore <<
766 /// use kernel::assert_pinned; <<
767 /// #[pin_data] <<
768 /// struct MyStruct { <<
769 /// some_field: u64, <<
770 /// } <<
771 /// <<
772 /// assert_pinned!(MyStruct, some_field, u64); <<
773 /// ``` <<
774 /// <<
775 /// Some uses of the macro may trigger the `ca <<
776 /// work around this, you may pass the `inline <<
777 /// only be used when the macro is invoked fro <<
778 /// ``` <<
779 /// use kernel::assert_pinned; <<
780 /// #[pin_data] <<
781 /// struct Foo<T> { <<
782 /// #[pin] <<
783 /// elem: T, <<
784 /// } <<
785 /// <<
786 /// impl<T> Foo<T> { <<
787 /// fn project(self: Pin<&mut Self>) -> Pi <<
788 /// assert_pinned!(Foo<T>, elem, T, in <<
789 /// <<
790 /// // SAFETY: The field is structural <<
791 /// unsafe { self.map_unchecked_mut(|m <<
792 /// } <<
793 /// } <<
794 /// ``` <<
795 #[macro_export] <<
796 macro_rules! assert_pinned { <<
797 ($ty:ty, $field:ident, $field_ty:ty, inlin <<
798 let _ = move |ptr: *mut $field_ty| { <<
799 // SAFETY: This code is unreachabl <<
800 let data = unsafe { <$ty as $crate <<
801 let init = $crate::init::__interna <<
802 // SAFETY: This code is unreachabl <<
803 unsafe { data.$field(ptr, init) }. <<
804 }; 897 };
>> 898 // Ensure that `data` really is of type `InitData` and help with type inference:
>> 899 let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>(
>> 900 data,
>> 901 move |slot| {
>> 902 {
>> 903 // Shadow the structure so it cannot be used to return early.
>> 904 struct __InitOk;
>> 905 // Create the `this` so it can be referenced by the user inside of the
>> 906 // expressions creating the individual fields.
>> 907 $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
>> 908 // Initialize every field.
>> 909 $crate::try_init!(init_slot:
>> 910 @slot(slot),
>> 911 @munch_fields($($fields)*,),
>> 912 );
>> 913 // We use unreachable code to ensure that all fields have been mentioned exactly
>> 914 // once, this struct initializer will still be type-checked and complain with a
>> 915 // very natural error message if a field is forgotten/mentioned more than once.
>> 916 #[allow(unreachable_code, clippy::diverging_sub_expression)]
>> 917 if false {
>> 918 $crate::try_init!(make_initializer:
>> 919 @slot(slot),
>> 920 @type_name($t),
>> 921 @munch_fields($($fields)*,),
>> 922 @acc(),
>> 923 );
>> 924 }
>> 925 // Forget all guards, since initialization was a success.
>> 926 $crate::try_init!(forget_guards:
>> 927 @munch_fields($($fields)*,),
>> 928 );
>> 929 }
>> 930 Ok(__InitOk)
>> 931 }
>> 932 );
>> 933 let init = move |slot| -> ::core::result::Result<(), $err> {
>> 934 init(slot).map(|__InitOk| ())
>> 935 };
>> 936 let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) };
>> 937 init
>> 938 }};
>> 939 (init_slot:
>> 940 @slot($slot:ident),
>> 941 @munch_fields( $(,)?),
>> 942 ) => {
>> 943 // Endpoint of munching, no fields are left.
805 }; 944 };
>> 945 (init_slot:
>> 946 @slot($slot:ident),
>> 947 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
>> 948 ) => {
>> 949 let $field = $val;
>> 950 // Call the initializer.
>> 951 //
>> 952 // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
>> 953 // return when an error/panic occurs.
>> 954 unsafe {
>> 955 $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?;
>> 956 }
>> 957 // Create the drop guard.
>> 958 //
>> 959 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
>> 960 //
>> 961 // SAFETY: We forget the guard later when initialization has succeeded.
>> 962 let $field = &unsafe {
>> 963 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
>> 964 };
806 965
807 ($ty:ty, $field:ident, $field_ty:ty) => { !! 966 $crate::try_init!(init_slot:
808 const _: () = { !! 967 @slot($slot),
809 $crate::assert_pinned!($ty, $field !! 968 @munch_fields($($rest)*),
>> 969 );
>> 970 };
>> 971 (init_slot:
>> 972 @slot($slot:ident),
>> 973 // Direct value init.
>> 974 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
>> 975 ) => {
>> 976 $(let $field = $val;)?
>> 977 // Call the initializer.
>> 978 //
>> 979 // SAFETY: The memory at `slot` is uninitialized.
>> 980 unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
>> 981 // Create the drop guard.
>> 982 //
>> 983 // We only give access to `&DropGuard`, so it cannot be accidentally forgotten.
>> 984 //
>> 985 // SAFETY: We forget the guard later when initialization has succeeded.
>> 986 let $field = &unsafe {
>> 987 $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
810 }; 988 };
>> 989
>> 990 $crate::try_init!(init_slot:
>> 991 @slot($slot),
>> 992 @munch_fields($($rest)*),
>> 993 );
>> 994 };
>> 995 (make_initializer:
>> 996 @slot($slot:ident),
>> 997 @type_name($t:ident),
>> 998 @munch_fields( $(,)?),
>> 999 @acc($($acc:tt)*),
>> 1000 ) => {
>> 1001 // Endpoint, nothing more to munch, create the initializer.
>> 1002 // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to
>> 1003 // get the correct type inference here:
>> 1004 unsafe {
>> 1005 ::core::ptr::write($slot, $t {
>> 1006 $($acc)*
>> 1007 });
>> 1008 }
>> 1009 };
>> 1010 (make_initializer:
>> 1011 @slot($slot:ident),
>> 1012 @type_name($t:ident),
>> 1013 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
>> 1014 @acc($($acc:tt)*),
>> 1015 ) => {
>> 1016 $crate::try_init!(make_initializer:
>> 1017 @slot($slot),
>> 1018 @type_name($t),
>> 1019 @munch_fields($($rest)*),
>> 1020 @acc($($acc)*$field: ::core::panic!(),),
>> 1021 );
>> 1022 };
>> 1023 (make_initializer:
>> 1024 @slot($slot:ident),
>> 1025 @type_name($t:ident),
>> 1026 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
>> 1027 @acc($($acc:tt)*),
>> 1028 ) => {
>> 1029 $crate::try_init!(make_initializer:
>> 1030 @slot($slot),
>> 1031 @type_name($t),
>> 1032 @munch_fields($($rest)*),
>> 1033 @acc($($acc)*$field: ::core::panic!(),),
>> 1034 );
>> 1035 };
>> 1036 (forget_guards:
>> 1037 @munch_fields($(,)?),
>> 1038 ) => {
>> 1039 // Munching finished.
>> 1040 };
>> 1041 (forget_guards:
>> 1042 @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
>> 1043 ) => {
>> 1044 unsafe { $crate::init::__internal::DropGuard::forget($field) };
>> 1045
>> 1046 $crate::try_init!(forget_guards:
>> 1047 @munch_fields($($rest)*),
>> 1048 );
>> 1049 };
>> 1050 (forget_guards:
>> 1051 @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
>> 1052 ) => {
>> 1053 unsafe { $crate::init::__internal::DropGuard::forget($field) };
>> 1054
>> 1055 $crate::try_init!(forget_guards:
>> 1056 @munch_fields($($rest)*),
>> 1057 );
811 }; 1058 };
812 } 1059 }
813 1060
814 /// A pin-initializer for the type `T`. 1061 /// A pin-initializer for the type `T`.
815 /// 1062 ///
816 /// To use this initializer, you will need a s 1063 /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
817 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>` 1064 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
818 /// [`InPlaceInit::pin_init`] function of a sm 1065 /// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
819 /// 1066 ///
820 /// Also see the [module description](self). 1067 /// Also see the [module description](self).
821 /// 1068 ///
822 /// # Safety 1069 /// # Safety
823 /// 1070 ///
824 /// When implementing this trait you will need !! 1071 /// When implementing this type you will need to take great care. Also there are probably very few
825 /// cases where a manual implementation is nec 1072 /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
826 /// 1073 ///
827 /// The [`PinInit::__pinned_init`] function: !! 1074 /// The [`PinInit::__pinned_init`] function
828 /// - returns `Ok(())` if it initialized every 1075 /// - returns `Ok(())` if it initialized every field of `slot`,
829 /// - returns `Err(err)` if it encountered an 1076 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
830 /// - `slot` can be deallocated without UB 1077 /// - `slot` can be deallocated without UB occurring,
831 /// - `slot` does not need to be dropped, 1078 /// - `slot` does not need to be dropped,
832 /// - `slot` is not partially initialized. 1079 /// - `slot` is not partially initialized.
833 /// - while constructing the `T` at `slot` it 1080 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
834 /// 1081 ///
835 /// [`Arc<T>`]: crate::sync::Arc 1082 /// [`Arc<T>`]: crate::sync::Arc
836 /// [`Arc::pin_init`]: crate::sync::Arc::pin_i 1083 /// [`Arc::pin_init`]: crate::sync::Arc::pin_init
837 #[must_use = "An initializer must be used in o 1084 #[must_use = "An initializer must be used in order to create its value."]
838 pub unsafe trait PinInit<T: ?Sized, E = Infall 1085 pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
839 /// Initializes `slot`. 1086 /// Initializes `slot`.
840 /// 1087 ///
841 /// # Safety 1088 /// # Safety
842 /// 1089 ///
843 /// - `slot` is a valid pointer to uniniti 1090 /// - `slot` is a valid pointer to uninitialized memory.
844 /// - the caller does not touch `slot` whe 1091 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
845 /// deallocate. 1092 /// deallocate.
846 /// - `slot` will not move until it is dro 1093 /// - `slot` will not move until it is dropped, i.e. it will be pinned.
847 unsafe fn __pinned_init(self, slot: *mut T 1094 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
848 <<
849 /// First initializes the value using `sel <<
850 /// value. <<
851 /// <<
852 /// If `f` returns an error the value is d <<
853 /// <<
854 /// # Examples <<
855 /// <<
856 /// ```rust <<
857 /// # #![allow(clippy::disallowed_names)] <<
858 /// use kernel::{types::Opaque, init::pin_ <<
859 /// #[repr(C)] <<
860 /// struct RawFoo([u8; 16]); <<
861 /// extern { <<
862 /// fn init_foo(_: *mut RawFoo); <<
863 /// } <<
864 /// <<
865 /// #[pin_data] <<
866 /// struct Foo { <<
867 /// #[pin] <<
868 /// raw: Opaque<RawFoo>, <<
869 /// } <<
870 /// <<
871 /// impl Foo { <<
872 /// fn setup(self: Pin<&mut Self>) { <<
873 /// pr_info!("Setting up foo"); <<
874 /// } <<
875 /// } <<
876 /// <<
877 /// let foo = pin_init!(Foo { <<
878 /// raw <- unsafe { <<
879 /// Opaque::ffi_init(|s| { <<
880 /// init_foo(s); <<
881 /// }) <<
882 /// }, <<
883 /// }).pin_chain(|foo| { <<
884 /// foo.setup(); <<
885 /// Ok(()) <<
886 /// }); <<
887 /// ``` <<
888 fn pin_chain<F>(self, f: F) -> ChainPinIni <<
889 where <<
890 F: FnOnce(Pin<&mut T>) -> Result<(), E <<
891 { <<
892 ChainPinInit(self, f, PhantomData) <<
893 } <<
894 } <<
895 <<
896 /// An initializer returned by [`PinInit::pin_ <<
897 pub struct ChainPinInit<I, F, T: ?Sized, E>(I, <<
898 <<
899 // SAFETY: The `__pinned_init` function is imp <<
900 // - returns `Ok(())` on successful initializa <<
901 // - returns `Err(err)` on error and in this c <<
902 // - considers `slot` pinned. <<
903 unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> <<
904 where <<
905 I: PinInit<T, E>, <<
906 F: FnOnce(Pin<&mut T>) -> Result<(), E>, <<
907 { <<
908 unsafe fn __pinned_init(self, slot: *mut T <<
909 // SAFETY: All requirements fulfilled <<
910 unsafe { self.0.__pinned_init(slot)? } <<
911 // SAFETY: The above call initialized <<
912 let val = unsafe { &mut *slot }; <<
913 // SAFETY: `slot` is considered pinned <<
914 let val = unsafe { Pin::new_unchecked( <<
915 // SAFETY: `slot` was initialized abov <<
916 (self.1)(val).inspect_err(|_| unsafe { <<
917 } <<
918 } 1095 }
919 1096
920 /// An initializer for `T`. 1097 /// An initializer for `T`.
921 /// 1098 ///
922 /// To use this initializer, you will need a s 1099 /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
923 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>` 1100 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the
924 /// [`InPlaceInit::init`] function of a smart 1101 /// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
925 /// [`PinInit<T, E>`] is a super trait, you ca 1102 /// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
926 /// 1103 ///
927 /// Also see the [module description](self). 1104 /// Also see the [module description](self).
928 /// 1105 ///
929 /// # Safety 1106 /// # Safety
930 /// 1107 ///
931 /// When implementing this trait you will need !! 1108 /// When implementing this type you will need to take great care. Also there are probably very few
932 /// cases where a manual implementation is nec 1109 /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
933 /// 1110 ///
934 /// The [`Init::__init`] function: !! 1111 /// The [`Init::__init`] function
935 /// - returns `Ok(())` if it initialized every 1112 /// - returns `Ok(())` if it initialized every field of `slot`,
936 /// - returns `Err(err)` if it encountered an 1113 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
937 /// - `slot` can be deallocated without UB 1114 /// - `slot` can be deallocated without UB occurring,
938 /// - `slot` does not need to be dropped, 1115 /// - `slot` does not need to be dropped,
939 /// - `slot` is not partially initialized. 1116 /// - `slot` is not partially initialized.
940 /// - while constructing the `T` at `slot` it 1117 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
941 /// 1118 ///
942 /// The `__pinned_init` function from the supe 1119 /// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
943 /// code as `__init`. 1120 /// code as `__init`.
944 /// 1121 ///
945 /// Contrary to its supertype [`PinInit<T, E>` 1122 /// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
946 /// move the pointee after initialization. 1123 /// move the pointee after initialization.
947 /// 1124 ///
948 /// [`Arc<T>`]: crate::sync::Arc 1125 /// [`Arc<T>`]: crate::sync::Arc
949 #[must_use = "An initializer must be used in o 1126 #[must_use = "An initializer must be used in order to create its value."]
950 pub unsafe trait Init<T: ?Sized, E = Infallibl !! 1127 pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized {
951 /// Initializes `slot`. 1128 /// Initializes `slot`.
952 /// 1129 ///
953 /// # Safety 1130 /// # Safety
954 /// 1131 ///
955 /// - `slot` is a valid pointer to uniniti 1132 /// - `slot` is a valid pointer to uninitialized memory.
956 /// - the caller does not touch `slot` whe 1133 /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
957 /// deallocate. 1134 /// deallocate.
958 unsafe fn __init(self, slot: *mut T) -> Re 1135 unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
959 <<
960 /// First initializes the value using `sel <<
961 /// value. <<
962 /// <<
963 /// If `f` returns an error the value is d <<
964 /// <<
965 /// # Examples <<
966 /// <<
967 /// ```rust <<
968 /// # #![allow(clippy::disallowed_names)] <<
969 /// use kernel::{types::Opaque, init::{sel <<
970 /// struct Foo { <<
971 /// buf: [u8; 1_000_000], <<
972 /// } <<
973 /// <<
974 /// impl Foo { <<
975 /// fn setup(&mut self) { <<
976 /// pr_info!("Setting up foo"); <<
977 /// } <<
978 /// } <<
979 /// <<
980 /// let foo = init!(Foo { <<
981 /// buf <- init::zeroed() <<
982 /// }).chain(|foo| { <<
983 /// foo.setup(); <<
984 /// Ok(()) <<
985 /// }); <<
986 /// ``` <<
987 fn chain<F>(self, f: F) -> ChainInit<Self, <<
988 where <<
989 F: FnOnce(&mut T) -> Result<(), E>, <<
990 { <<
991 ChainInit(self, f, PhantomData) <<
992 } <<
993 } <<
994 <<
995 /// An initializer returned by [`Init::chain`] <<
996 pub struct ChainInit<I, F, T: ?Sized, E>(I, F, <<
997 <<
998 // SAFETY: The `__init` function is implemente <<
999 // - returns `Ok(())` on successful initializa <<
1000 // - returns `Err(err)` on error and in this <<
1001 unsafe impl<T: ?Sized, E, I, F> Init<T, E> fo <<
1002 where <<
1003 I: Init<T, E>, <<
1004 F: FnOnce(&mut T) -> Result<(), E>, <<
1005 { <<
1006 unsafe fn __init(self, slot: *mut T) -> R <<
1007 // SAFETY: All requirements fulfilled <<
1008 unsafe { self.0.__pinned_init(slot)? <<
1009 // SAFETY: The above call initialized <<
1010 (self.1)(unsafe { &mut *slot }).inspe <<
1011 // SAFETY: `slot` was initialized <<
1012 unsafe { core::ptr::drop_in_place <<
1013 } <<
1014 } 1136 }
1015 1137
1016 // SAFETY: `__pinned_init` behaves exactly th !! 1138 // SAFETY: Every in-place initializer can also be used as a pin-initializer.
1017 unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> !! 1139 unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I
1018 where 1140 where
1019 I: Init<T, E>, 1141 I: Init<T, E>,
1020 F: FnOnce(&mut T) -> Result<(), E>, <<
1021 { 1142 {
1022 unsafe fn __pinned_init(self, slot: *mut 1143 unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
1023 // SAFETY: `__init` has less strict r !! 1144 // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not
>> 1145 // require `slot` to not move after init.
1024 unsafe { self.__init(slot) } 1146 unsafe { self.__init(slot) }
1025 } 1147 }
1026 } 1148 }
1027 1149
1028 /// Creates a new [`PinInit<T, E>`] from the 1150 /// Creates a new [`PinInit<T, E>`] from the given closure.
1029 /// 1151 ///
1030 /// # Safety 1152 /// # Safety
1031 /// 1153 ///
1032 /// The closure: 1154 /// The closure:
1033 /// - returns `Ok(())` if it initialized ever 1155 /// - returns `Ok(())` if it initialized every field of `slot`,
1034 /// - returns `Err(err)` if it encountered an 1156 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1035 /// - `slot` can be deallocated without U 1157 /// - `slot` can be deallocated without UB occurring,
1036 /// - `slot` does not need to be dropped, 1158 /// - `slot` does not need to be dropped,
1037 /// - `slot` is not partially initialized 1159 /// - `slot` is not partially initialized.
1038 /// - may assume that the `slot` does not mov 1160 /// - may assume that the `slot` does not move if `T: !Unpin`,
1039 /// - while constructing the `T` at `slot` it 1161 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1040 #[inline] 1162 #[inline]
1041 pub const unsafe fn pin_init_from_closure<T: 1163 pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
1042 f: impl FnOnce(*mut T) -> Result<(), E>, 1164 f: impl FnOnce(*mut T) -> Result<(), E>,
1043 ) -> impl PinInit<T, E> { 1165 ) -> impl PinInit<T, E> {
1044 __internal::InitClosure(f, PhantomData) 1166 __internal::InitClosure(f, PhantomData)
1045 } 1167 }
1046 1168
1047 /// Creates a new [`Init<T, E>`] from the giv 1169 /// Creates a new [`Init<T, E>`] from the given closure.
1048 /// 1170 ///
1049 /// # Safety 1171 /// # Safety
1050 /// 1172 ///
1051 /// The closure: 1173 /// The closure:
1052 /// - returns `Ok(())` if it initialized ever 1174 /// - returns `Ok(())` if it initialized every field of `slot`,
1053 /// - returns `Err(err)` if it encountered an 1175 /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
1054 /// - `slot` can be deallocated without U 1176 /// - `slot` can be deallocated without UB occurring,
1055 /// - `slot` does not need to be dropped, 1177 /// - `slot` does not need to be dropped,
1056 /// - `slot` is not partially initialized 1178 /// - `slot` is not partially initialized.
1057 /// - the `slot` may move after initializatio 1179 /// - the `slot` may move after initialization.
1058 /// - while constructing the `T` at `slot` it 1180 /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
1059 #[inline] 1181 #[inline]
1060 pub const unsafe fn init_from_closure<T: ?Siz 1182 pub const unsafe fn init_from_closure<T: ?Sized, E>(
1061 f: impl FnOnce(*mut T) -> Result<(), E>, 1183 f: impl FnOnce(*mut T) -> Result<(), E>,
1062 ) -> impl Init<T, E> { 1184 ) -> impl Init<T, E> {
1063 __internal::InitClosure(f, PhantomData) 1185 __internal::InitClosure(f, PhantomData)
1064 } 1186 }
1065 1187
1066 /// An initializer that leaves the memory uni 1188 /// An initializer that leaves the memory uninitialized.
1067 /// 1189 ///
1068 /// The initializer is a no-op. The `slot` me 1190 /// The initializer is a no-op. The `slot` memory is not changed.
1069 #[inline] 1191 #[inline]
1070 pub fn uninit<T, E>() -> impl Init<MaybeUnini 1192 pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
1071 // SAFETY: The memory is allowed to be un 1193 // SAFETY: The memory is allowed to be uninitialized.
1072 unsafe { init_from_closure(|_| Ok(())) } 1194 unsafe { init_from_closure(|_| Ok(())) }
1073 } 1195 }
1074 1196
1075 /// Initializes an array by initializing each <<
1076 /// <<
1077 /// # Examples <<
1078 /// <<
1079 /// ```rust <<
1080 /// use kernel::{error::Error, init::init_arr <<
1081 /// let array: Box<[usize; 1_000]> = Box::ini <<
1082 /// assert_eq!(array.len(), 1_000); <<
1083 /// ``` <<
1084 pub fn init_array_from_fn<I, const N: usize, <<
1085 mut make_init: impl FnMut(usize) -> I, <<
1086 ) -> impl Init<[T; N], E> <<
1087 where <<
1088 I: Init<T, E>, <<
1089 { <<
1090 let init = move |slot: *mut [T; N]| { <<
1091 let slot = slot.cast::<T>(); <<
1092 // Counts the number of initialized e <<
1093 // `slot`. <<
1094 let mut init_count = ScopeGuard::new_ <<
1095 // We now free every element that <<
1096 // SAFETY: The loop initialized e <<
1097 // return `Err` below, the caller <<
1098 // uninitialized. <<
1099 unsafe { ptr::drop_in_place(ptr:: <<
1100 }); <<
1101 for i in 0..N { <<
1102 let init = make_init(i); <<
1103 // SAFETY: Since 0 <= `i` < N, it <<
1104 let ptr = unsafe { slot.add(i) }; <<
1105 // SAFETY: The pointer is derived <<
1106 // requirements. <<
1107 unsafe { init.__init(ptr) }?; <<
1108 *init_count += 1; <<
1109 } <<
1110 init_count.dismiss(); <<
1111 Ok(()) <<
1112 }; <<
1113 // SAFETY: The initializer above initiali <<
1114 // any initialized elements and returns ` <<
1115 unsafe { init_from_closure(init) } <<
1116 } <<
1117 <<
1118 /// Initializes an array by initializing each <<
1119 /// <<
1120 /// # Examples <<
1121 /// <<
1122 /// ```rust <<
1123 /// use kernel::{sync::{Arc, Mutex}, init::pi <<
1124 /// let array: Arc<[Mutex<usize>; 1_000]> = <<
1125 /// Arc::pin_init(pin_init_array_from_fn( <<
1126 /// assert_eq!(array.len(), 1_000); <<
1127 /// ``` <<
1128 pub fn pin_init_array_from_fn<I, const N: usi <<
1129 mut make_init: impl FnMut(usize) -> I, <<
1130 ) -> impl PinInit<[T; N], E> <<
1131 where <<
1132 I: PinInit<T, E>, <<
1133 { <<
1134 let init = move |slot: *mut [T; N]| { <<
1135 let slot = slot.cast::<T>(); <<
1136 // Counts the number of initialized e <<
1137 // `slot`. <<
1138 let mut init_count = ScopeGuard::new_ <<
1139 // We now free every element that <<
1140 // SAFETY: The loop initialized e <<
1141 // return `Err` below, the caller <<
1142 // uninitialized. <<
1143 unsafe { ptr::drop_in_place(ptr:: <<
1144 }); <<
1145 for i in 0..N { <<
1146 let init = make_init(i); <<
1147 // SAFETY: Since 0 <= `i` < N, it <<
1148 let ptr = unsafe { slot.add(i) }; <<
1149 // SAFETY: The pointer is derived <<
1150 // requirements. <<
1151 unsafe { init.__pinned_init(ptr) <<
1152 *init_count += 1; <<
1153 } <<
1154 init_count.dismiss(); <<
1155 Ok(()) <<
1156 }; <<
1157 // SAFETY: The initializer above initiali <<
1158 // any initialized elements and returns ` <<
1159 unsafe { pin_init_from_closure(init) } <<
1160 } <<
1161 <<
1162 // SAFETY: Every type can be initialized by-v 1197 // SAFETY: Every type can be initialized by-value.
1163 unsafe impl<T, E> Init<T, E> for T { 1198 unsafe impl<T, E> Init<T, E> for T {
1164 unsafe fn __init(self, slot: *mut T) -> R 1199 unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
1165 unsafe { slot.write(self) }; 1200 unsafe { slot.write(self) };
1166 Ok(()) 1201 Ok(())
1167 } 1202 }
1168 } 1203 }
1169 1204
1170 // SAFETY: Every type can be initialized by-v <<
1171 unsafe impl<T, E> PinInit<T, E> for T { <<
1172 unsafe fn __pinned_init(self, slot: *mut <<
1173 unsafe { self.__init(slot) } <<
1174 } <<
1175 } <<
1176 <<
1177 /// Smart pointer that can initialize memory 1205 /// Smart pointer that can initialize memory in-place.
1178 pub trait InPlaceInit<T>: Sized { 1206 pub trait InPlaceInit<T>: Sized {
1179 /// Pinned version of `Self`. <<
1180 /// <<
1181 /// If a type already implicitly pins its <<
1182 /// `Self`, otherwise just use `Pin<Self> <<
1183 type PinnedSelf; <<
1184 <<
1185 /// Use the given pin-initializer to pin- 1207 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1186 /// type. 1208 /// type.
1187 /// 1209 ///
1188 /// If `T: !Unpin` it will not be able to 1210 /// If `T: !Unpin` it will not be able to move afterwards.
1189 fn try_pin_init<E>(init: impl PinInit<T, !! 1211 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1190 where 1212 where
1191 E: From<AllocError>; 1213 E: From<AllocError>;
1192 1214
1193 /// Use the given pin-initializer to pin- 1215 /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
1194 /// type. 1216 /// type.
1195 /// 1217 ///
1196 /// If `T: !Unpin` it will not be able to 1218 /// If `T: !Unpin` it will not be able to move afterwards.
1197 fn pin_init<E>(init: impl PinInit<T, E>, !! 1219 fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>>
1198 where 1220 where
1199 Error: From<E>, 1221 Error: From<E>,
1200 { 1222 {
1201 // SAFETY: We delegate to `init` and 1223 // SAFETY: We delegate to `init` and only change the error type.
1202 let init = unsafe { 1224 let init = unsafe {
1203 pin_init_from_closure(|slot| init 1225 pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1204 }; 1226 };
1205 Self::try_pin_init(init, flags) !! 1227 Self::try_pin_init(init)
1206 } 1228 }
1207 1229
1208 /// Use the given initializer to in-place 1230 /// Use the given initializer to in-place initialize a `T`.
1209 fn try_init<E>(init: impl Init<T, E>, fla !! 1231 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1210 where 1232 where
1211 E: From<AllocError>; 1233 E: From<AllocError>;
1212 1234
1213 /// Use the given initializer to in-place 1235 /// Use the given initializer to in-place initialize a `T`.
1214 fn init<E>(init: impl Init<T, E>, flags: !! 1236 fn init<E>(init: impl Init<T, E>) -> error::Result<Self>
1215 where 1237 where
1216 Error: From<E>, 1238 Error: From<E>,
1217 { 1239 {
1218 // SAFETY: We delegate to `init` and 1240 // SAFETY: We delegate to `init` and only change the error type.
1219 let init = unsafe { 1241 let init = unsafe {
1220 init_from_closure(|slot| init.__p 1242 init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
1221 }; 1243 };
1222 Self::try_init(init, flags) !! 1244 Self::try_init(init)
1223 } <<
1224 } <<
1225 <<
1226 impl<T> InPlaceInit<T> for Arc<T> { <<
1227 type PinnedSelf = Self; <<
1228 <<
1229 #[inline] <<
1230 fn try_pin_init<E>(init: impl PinInit<T, <<
1231 where <<
1232 E: From<AllocError>, <<
1233 { <<
1234 UniqueArc::try_pin_init(init, flags). <<
1235 } <<
1236 <<
1237 #[inline] <<
1238 fn try_init<E>(init: impl Init<T, E>, fla <<
1239 where <<
1240 E: From<AllocError>, <<
1241 { <<
1242 UniqueArc::try_init(init, flags).map( <<
1243 } 1245 }
1244 } 1246 }
1245 1247
1246 impl<T> InPlaceInit<T> for Box<T> { 1248 impl<T> InPlaceInit<T> for Box<T> {
1247 type PinnedSelf = Pin<Self>; <<
1248 <<
1249 #[inline] 1249 #[inline]
1250 fn try_pin_init<E>(init: impl PinInit<T, !! 1250 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1251 where 1251 where
1252 E: From<AllocError>, 1252 E: From<AllocError>,
1253 { 1253 {
1254 <Box<_> as BoxExt<_>>::new_uninit(fla !! 1254 let mut this = Box::try_new_uninit()?;
>> 1255 let slot = this.as_mut_ptr();
>> 1256 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
>> 1257 // slot is valid and will not be moved, because we pin it later.
>> 1258 unsafe { init.__pinned_init(slot)? };
>> 1259 // SAFETY: All fields have been initialized.
>> 1260 Ok(unsafe { this.assume_init() }.into())
1255 } 1261 }
1256 1262
1257 #[inline] 1263 #[inline]
1258 fn try_init<E>(init: impl Init<T, E>, fla !! 1264 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
1259 where 1265 where
1260 E: From<AllocError>, 1266 E: From<AllocError>,
1261 { 1267 {
1262 <Box<_> as BoxExt<_>>::new_uninit(fla !! 1268 let mut this = Box::try_new_uninit()?;
>> 1269 let slot = this.as_mut_ptr();
>> 1270 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
>> 1271 // slot is valid.
>> 1272 unsafe { init.__init(slot)? };
>> 1273 // SAFETY: All fields have been initialized.
>> 1274 Ok(unsafe { this.assume_init() })
1263 } 1275 }
1264 } 1276 }
1265 1277
1266 impl<T> InPlaceInit<T> for UniqueArc<T> { 1278 impl<T> InPlaceInit<T> for UniqueArc<T> {
1267 type PinnedSelf = Pin<Self>; <<
1268 <<
1269 #[inline] <<
1270 fn try_pin_init<E>(init: impl PinInit<T, <<
1271 where <<
1272 E: From<AllocError>, <<
1273 { <<
1274 UniqueArc::new_uninit(flags)?.write_p <<
1275 } <<
1276 <<
1277 #[inline] 1279 #[inline]
1278 fn try_init<E>(init: impl Init<T, E>, fla !! 1280 fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E>
1279 where 1281 where
1280 E: From<AllocError>, 1282 E: From<AllocError>,
1281 { 1283 {
1282 UniqueArc::new_uninit(flags)?.write_i !! 1284 let mut this = UniqueArc::try_new_uninit()?;
1283 } !! 1285 let slot = this.as_mut_ptr();
1284 } <<
1285 <<
1286 /// Smart pointer containing uninitialized me <<
1287 pub trait InPlaceWrite<T> { <<
1288 /// The type `Self` turns into when the c <<
1289 type Initialized; <<
1290 <<
1291 /// Use the given initializer to write a <<
1292 /// <<
1293 /// Does not drop the current value and c <<
1294 fn write_init<E>(self, init: impl Init<T, <<
1295 <<
1296 /// Use the given pin-initializer to writ <<
1297 /// <<
1298 /// Does not drop the current value and c <<
1299 fn write_pin_init<E>(self, init: impl Pin <<
1300 } <<
1301 <<
1302 impl<T> InPlaceWrite<T> for Box<MaybeUninit<T <<
1303 type Initialized = Box<T>; <<
1304 <<
1305 fn write_init<E>(mut self, init: impl Ini <<
1306 let slot = self.as_mut_ptr(); <<
1307 // SAFETY: When init errors/panics, s <<
1308 // slot is valid. <<
1309 unsafe { init.__init(slot)? }; <<
1310 // SAFETY: All fields have been initi <<
1311 Ok(unsafe { self.assume_init() }) <<
1312 } <<
1313 <<
1314 fn write_pin_init<E>(mut self, init: impl <<
1315 let slot = self.as_mut_ptr(); <<
1316 // SAFETY: When init errors/panics, s 1286 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1317 // slot is valid and will not be move 1287 // slot is valid and will not be moved, because we pin it later.
1318 unsafe { init.__pinned_init(slot)? }; 1288 unsafe { init.__pinned_init(slot)? };
1319 // SAFETY: All fields have been initi 1289 // SAFETY: All fields have been initialized.
1320 Ok(unsafe { self.assume_init() }.into !! 1290 Ok(unsafe { this.assume_init() }.into())
1321 } 1291 }
1322 } <<
1323 <<
1324 impl<T> InPlaceWrite<T> for UniqueArc<MaybeUn <<
1325 type Initialized = UniqueArc<T>; <<
1326 1292
1327 fn write_init<E>(mut self, init: impl Ini !! 1293 #[inline]
1328 let slot = self.as_mut_ptr(); !! 1294 fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E>
>> 1295 where
>> 1296 E: From<AllocError>,
>> 1297 {
>> 1298 let mut this = UniqueArc::try_new_uninit()?;
>> 1299 let slot = this.as_mut_ptr();
1329 // SAFETY: When init errors/panics, s 1300 // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
1330 // slot is valid. 1301 // slot is valid.
1331 unsafe { init.__init(slot)? }; 1302 unsafe { init.__init(slot)? };
1332 // SAFETY: All fields have been initi 1303 // SAFETY: All fields have been initialized.
1333 Ok(unsafe { self.assume_init() }) !! 1304 Ok(unsafe { this.assume_init() })
1334 } <<
1335 <<
1336 fn write_pin_init<E>(mut self, init: impl <<
1337 let slot = self.as_mut_ptr(); <<
1338 // SAFETY: When init errors/panics, s <<
1339 // slot is valid and will not be move <<
1340 unsafe { init.__pinned_init(slot)? }; <<
1341 // SAFETY: All fields have been initi <<
1342 Ok(unsafe { self.assume_init() }.into <<
1343 } 1305 }
1344 } 1306 }
1345 1307
1346 /// Trait facilitating pinned destruction. 1308 /// Trait facilitating pinned destruction.
1347 /// 1309 ///
1348 /// Use [`pinned_drop`] to implement this tra 1310 /// Use [`pinned_drop`] to implement this trait safely:
1349 /// 1311 ///
1350 /// ```rust 1312 /// ```rust
1351 /// # use kernel::sync::Mutex; 1313 /// # use kernel::sync::Mutex;
1352 /// use kernel::macros::pinned_drop; 1314 /// use kernel::macros::pinned_drop;
1353 /// use core::pin::Pin; 1315 /// use core::pin::Pin;
1354 /// #[pin_data(PinnedDrop)] 1316 /// #[pin_data(PinnedDrop)]
1355 /// struct Foo { 1317 /// struct Foo {
1356 /// #[pin] 1318 /// #[pin]
1357 /// mtx: Mutex<usize>, 1319 /// mtx: Mutex<usize>,
1358 /// } 1320 /// }
1359 /// 1321 ///
1360 /// #[pinned_drop] 1322 /// #[pinned_drop]
1361 /// impl PinnedDrop for Foo { 1323 /// impl PinnedDrop for Foo {
1362 /// fn drop(self: Pin<&mut Self>) { 1324 /// fn drop(self: Pin<&mut Self>) {
1363 /// pr_info!("Foo is being dropped!") 1325 /// pr_info!("Foo is being dropped!");
1364 /// } 1326 /// }
1365 /// } 1327 /// }
1366 /// ``` 1328 /// ```
1367 /// 1329 ///
1368 /// # Safety 1330 /// # Safety
1369 /// 1331 ///
1370 /// This trait must be implemented via the [` 1332 /// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
1371 /// 1333 ///
1372 /// [`pinned_drop`]: kernel::macros::pinned_d 1334 /// [`pinned_drop`]: kernel::macros::pinned_drop
1373 pub unsafe trait PinnedDrop: __internal::HasP 1335 pub unsafe trait PinnedDrop: __internal::HasPinData {
1374 /// Executes the pinned destructor of thi 1336 /// Executes the pinned destructor of this type.
1375 /// 1337 ///
1376 /// While this function is marked safe, i 1338 /// While this function is marked safe, it is actually unsafe to call it manually. For this
1377 /// reason it takes an additional paramet 1339 /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
1378 /// and thus prevents this function from 1340 /// and thus prevents this function from being called where it should not.
1379 /// 1341 ///
1380 /// This extra parameter will be generate 1342 /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
1381 /// automatically. 1343 /// automatically.
1382 fn drop(self: Pin<&mut Self>, only_call_f 1344 fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
1383 } 1345 }
1384 1346
1385 /// Marker trait for types that can be initia 1347 /// Marker trait for types that can be initialized by writing just zeroes.
1386 /// 1348 ///
1387 /// # Safety 1349 /// # Safety
1388 /// 1350 ///
1389 /// The bit pattern consisting of only zeroes 1351 /// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
1390 /// this is not UB: 1352 /// this is not UB:
1391 /// 1353 ///
1392 /// ```rust,ignore 1354 /// ```rust,ignore
1393 /// let val: Self = unsafe { core::mem::zeroe 1355 /// let val: Self = unsafe { core::mem::zeroed() };
1394 /// ``` 1356 /// ```
1395 pub unsafe trait Zeroable {} 1357 pub unsafe trait Zeroable {}
1396 1358
1397 /// Create a new zeroed T. 1359 /// Create a new zeroed T.
1398 /// 1360 ///
1399 /// The returned initializer will write `0x00 1361 /// The returned initializer will write `0x00` to every byte of the given `slot`.
1400 #[inline] 1362 #[inline]
1401 pub fn zeroed<T: Zeroable>() -> impl Init<T> 1363 pub fn zeroed<T: Zeroable>() -> impl Init<T> {
1402 // SAFETY: Because `T: Zeroable`, all byt 1364 // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
1403 // and because we write all zeroes, the m 1365 // and because we write all zeroes, the memory is initialized.
1404 unsafe { 1366 unsafe {
1405 init_from_closure(|slot: *mut T| { 1367 init_from_closure(|slot: *mut T| {
1406 slot.write_bytes(0, 1); 1368 slot.write_bytes(0, 1);
1407 Ok(()) 1369 Ok(())
1408 }) 1370 })
1409 } 1371 }
1410 } 1372 }
1411 1373
1412 macro_rules! impl_zeroable { 1374 macro_rules! impl_zeroable {
1413 ($($({$($generics:tt)*})? $t:ty, )*) => { 1375 ($($({$($generics:tt)*})? $t:ty, )*) => {
1414 $(unsafe impl$($($generics)*)? Zeroab 1376 $(unsafe impl$($($generics)*)? Zeroable for $t {})*
1415 }; 1377 };
1416 } 1378 }
1417 1379
1418 impl_zeroable! { 1380 impl_zeroable! {
1419 // SAFETY: All primitives that are allowe 1381 // SAFETY: All primitives that are allowed to be zero.
1420 bool, 1382 bool,
1421 char, 1383 char,
1422 u8, u16, u32, u64, u128, usize, 1384 u8, u16, u32, u64, u128, usize,
1423 i8, i16, i32, i64, i128, isize, 1385 i8, i16, i32, i64, i128, isize,
1424 f32, f64, 1386 f32, f64,
1425 1387
1426 // Note: do not add uninhabited types (su !! 1388 // SAFETY: These are ZSTs, there is nothing to zero.
1427 // creating an instance of an uninhabited !! 1389 {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, Infallible, (),
1428 // uninhabited/empty types, consult The R <<
1429 // <https://doc.rust-lang.org/stable/nomi <<
1430 // also has information on undefined beha <<
1431 // <https://doc.rust-lang.org/stable/refe <<
1432 // <<
1433 // SAFETY: These are inhabited ZSTs; ther <<
1434 {<T: ?Sized>} PhantomData<T>, core::marke <<
1435 1390
1436 // SAFETY: Type is allowed to take any va 1391 // SAFETY: Type is allowed to take any value, including all zeros.
1437 {<T>} MaybeUninit<T>, 1392 {<T>} MaybeUninit<T>,
1438 // SAFETY: Type is allowed to take any va <<
1439 {<T>} Opaque<T>, <<
1440 <<
1441 // SAFETY: `T: Zeroable` and `UnsafeCell` <<
1442 {<T: ?Sized + Zeroable>} UnsafeCell<T>, <<
1443 1393
1444 // SAFETY: All zeros is equivalent to `No 1394 // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1445 Option<NonZeroU8>, Option<NonZeroU16>, Op 1395 Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
1446 Option<NonZeroU128>, Option<NonZeroUsize> 1396 Option<NonZeroU128>, Option<NonZeroUsize>,
1447 Option<NonZeroI8>, Option<NonZeroI16>, Op 1397 Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
1448 Option<NonZeroI128>, Option<NonZeroIsize> 1398 Option<NonZeroI128>, Option<NonZeroIsize>,
1449 1399
1450 // SAFETY: All zeros is equivalent to `No 1400 // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
1451 // 1401 //
1452 // In this case we are allowed to use `T: 1402 // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
1453 {<T: ?Sized>} Option<NonNull<T>>, 1403 {<T: ?Sized>} Option<NonNull<T>>,
1454 {<T: ?Sized>} Option<Box<T>>, 1404 {<T: ?Sized>} Option<Box<T>>,
1455 1405
1456 // SAFETY: `null` pointer is valid. 1406 // SAFETY: `null` pointer is valid.
1457 // 1407 //
1458 // We cannot use `T: ?Sized`, since the V 1408 // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
1459 // null. 1409 // null.
1460 // 1410 //
1461 // When `Pointee` gets stabilized, we cou 1411 // When `Pointee` gets stabilized, we could use
1462 // `T: ?Sized where <T as Pointee>::Metad 1412 // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
1463 {<T>} *mut T, {<T>} *const T, 1413 {<T>} *mut T, {<T>} *const T,
1464 1414
1465 // SAFETY: `null` pointer is valid and th 1415 // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
1466 // zero. 1416 // zero.
1467 {<T>} *mut [T], {<T>} *const [T], *mut st 1417 {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
1468 1418
1469 // SAFETY: `T` is `Zeroable`. 1419 // SAFETY: `T` is `Zeroable`.
1470 {<const N: usize, T: Zeroable>} [T; N], { 1420 {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
1471 } 1421 }
1472 1422
1473 macro_rules! impl_tuple_zeroable { 1423 macro_rules! impl_tuple_zeroable {
1474 ($(,)?) => {}; 1424 ($(,)?) => {};
1475 ($first:ident, $($t:ident),* $(,)?) => { 1425 ($first:ident, $($t:ident),* $(,)?) => {
1476 // SAFETY: All elements are zeroable 1426 // SAFETY: All elements are zeroable and padding can be zero.
1477 unsafe impl<$first: Zeroable, $($t: Z 1427 unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
1478 impl_tuple_zeroable!($($t),* ,); 1428 impl_tuple_zeroable!($($t),* ,);
1479 } 1429 }
1480 } 1430 }
1481 1431
1482 impl_tuple_zeroable!(A, B, C, D, E, F, G, H, 1432 impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.