1 // SPDX-License-Identifier: Apache-2.0 OR MIT
2
3 //! API to safely and fallibly initialize pinn
4 //!
5 //! It also allows in-place initialization of
6 //! overflow.
7 //!
8 //! Most `struct`s from the [`sync`] module ne
9 //! `struct`s from C. [Pinning][pinning] is Ru
10 //!
11 //! # Overview
12 //!
13 //! To initialize a `struct` with an in-place
14 //! - an in-place constructor,
15 //! - a memory location that can hold your `st
16 //! [`UniqueArc<T>`], [`Box<T>`] or any othe
17 //!
18 //! To get an in-place constructor there are g
19 //! - directly creating an in-place constructo
20 //! - a custom function/macro returning an in-
21 //! - using the unsafe function [`pin_init_fro
22 //!
23 //! Aside from pinned initialization, this API
24 //! the macros/types/functions are generally n
25 //! prefix.
26 //!
27 //! # Examples
28 //!
29 //! ## Using the [`pin_init!`] macro
30 //!
31 //! If you want to use [`PinInit`], then you w
32 //! `#[`[`pin_data`]`]`. It is a macro that us
33 //! [structurally pinned fields]. After doing
34 //! [`pin_init!`]. The syntax is almost the sa
35 //! that you need to write `<-` instead of `:`
36 //!
37 //! ```rust
38 //! # #![allow(clippy::disallowed_names)]
39 //! use kernel::sync::{new_mutex, Mutex};
40 //! # use core::pin::Pin;
41 //! #[pin_data]
42 //! struct Foo {
43 //! #[pin]
44 //! a: Mutex<usize>,
45 //! b: u32,
46 //! }
47 //!
48 //! let foo = pin_init!(Foo {
49 //! a <- new_mutex!(42, "Foo::a"),
50 //! b: 24,
51 //! });
52 //! ```
53 //!
54 //! `foo` now is of the type [`impl PinInit<Fo
55 //! (or just the stack) to actually initialize
56 //!
57 //! ```rust
58 //! # #![allow(clippy::disallowed_names)]
59 //! # use kernel::sync::{new_mutex, Mutex};
60 //! # use core::pin::Pin;
61 //! # #[pin_data]
62 //! # struct Foo {
63 //! # #[pin]
64 //! # a: Mutex<usize>,
65 //! # b: u32,
66 //! # }
67 //! # let foo = pin_init!(Foo {
68 //! # a <- new_mutex!(42, "Foo::a"),
69 //! # b: 24,
70 //! # });
71 //! let foo: Result<Pin<Box<Foo>>> = Box::pin_
72 //! ```
73 //!
74 //! For more information see the [`pin_init!`]
75 //!
76 //! ## Using a custom function/macro that retu
77 //!
78 //! Many types from the kernel supply a functi
79 //! above method only works for types where yo
80 //!
81 //! ```rust
82 //! # use kernel::sync::{new_mutex, Arc, Mutex
83 //! let mtx: Result<Arc<Mutex<usize>>> =
84 //! Arc::pin_init(new_mutex!(42, "example:
85 //! ```
86 //!
87 //! To declare an init macro/function you just
88 //!
89 //! ```rust
90 //! # #![allow(clippy::disallowed_names)]
91 //! # use kernel::{sync::Mutex, new_mutex, ini
92 //! #[pin_data]
93 //! struct DriverData {
94 //! #[pin]
95 //! status: Mutex<i32>,
96 //! buffer: Box<[u8; 1_000_000]>,
97 //! }
98 //!
99 //! impl DriverData {
100 //! fn new() -> impl PinInit<Self, Error>
101 //! try_pin_init!(Self {
102 //! status <- new_mutex!(0, "Drive
103 //! buffer: Box::init(kernel::init
104 //! })
105 //! }
106 //! }
107 //! ```
108 //!
109 //! ## Manual creation of an initializer
110 //!
111 //! Often when working with primitives the pre
112 //! [`pin_init_from_closure()`] comes in. This
113 //! [`impl PinInit<T, E>`] directly from a clo
114 //! actually does the initialization in the co
115 //! (we are calling the parameter to the closu
116 //! - when the closure returns `Ok(())`, then
117 //! `slot` now contains a valid bit pattern
118 //! - when the closure returns `Err(e)`, then
119 //! you need to take care to clean up anythi
120 //! - you may assume that `slot` will stay pin
121 //! `slot` gets called.
122 //!
123 //! ```rust
124 //! # #![allow(unreachable_pub, clippy::disall
125 //! use kernel::{init, types::Opaque};
126 //! use core::{ptr::addr_of_mut, marker::Phant
127 //! # mod bindings {
128 //! # #![allow(non_camel_case_types)]
129 //! # pub struct foo;
130 //! # pub unsafe fn init_foo(_ptr: *mut fo
131 //! # pub unsafe fn destroy_foo(_ptr: *mut
132 //! # pub unsafe fn enable_foo(_ptr: *mut
133 //! # }
134 //! # // `Error::from_errno` is `pub(crate)` i
135 //! # trait FromErrno {
136 //! # fn from_errno(errno: core::ffi::c_in
137 //! # // Dummy error that can be const
138 //! # Error::from(core::fmt::Error)
139 //! # }
140 //! # }
141 //! # impl FromErrno for Error {}
142 //! /// # Invariants
143 //! ///
144 //! /// `foo` is always initialized
145 //! #[pin_data(PinnedDrop)]
146 //! pub struct RawFoo {
147 //! #[pin]
148 //! foo: Opaque<bindings::foo>,
149 //! #[pin]
150 //! _p: PhantomPinned,
151 //! }
152 //!
153 //! impl RawFoo {
154 //! pub fn new(flags: u32) -> impl PinInit
155 //! // SAFETY:
156 //! // - when the closure returns `Ok(
157 //! // enabled `foo`,
158 //! // - when it returns `Err(e)`, the
159 //! unsafe {
160 //! init::pin_init_from_closure(mo
161 //! // `slot` contains uninit
162 //! let foo = addr_of_mut!((*s
163 //!
164 //! // Initialize the `foo`
165 //! bindings::init_foo(Opaque:
166 //!
167 //! // Try to enable it.
168 //! let err = bindings::enable
169 //! if err != 0 {
170 //! // Enabling has failed
171 //! bindings::destroy_foo(
172 //! return Err(Error::from
173 //! }
174 //!
175 //! // All fields of `RawFoo`
176 //! Ok(())
177 //! })
178 //! }
179 //! }
180 //! }
181 //!
182 //! #[pinned_drop]
183 //! impl PinnedDrop for RawFoo {
184 //! fn drop(self: Pin<&mut Self>) {
185 //! // SAFETY: Since `foo` is initiali
186 //! unsafe { bindings::destroy_foo(sel
187 //! }
188 //! }
189 //! ```
190 //!
191 //! For the special case where initializing a
192 //! there exist the helper function [`Opaque::
193 //! [`Opaque`] field by just delegating to the
194 //! with [`pin_init!`].
195 //!
196 //! For more information on how to use [`pin_i
197 //! the `kernel` crate. The [`sync`] module is
198 //!
199 //! [`sync`]: kernel::sync
200 //! [pinning]: https://doc.rust-lang.org/std/p
201 //! [structurally pinned fields]:
202 //! https://doc.rust-lang.org/std/pin/inde
203 //! [stack]: crate::stack_pin_init
204 //! [`Arc<T>`]: crate::sync::Arc
205 //! [`impl PinInit<Foo>`]: PinInit
206 //! [`impl PinInit<T, E>`]: PinInit
207 //! [`impl Init<T, E>`]: Init
208 //! [`Opaque`]: kernel::types::Opaque
209 //! [`Opaque::ffi_init`]: kernel::types::Opaqu
210 //! [`pin_data`]: ::macros::pin_data
211 //! [`pin_init!`]: crate::pin_init!
212
213 use crate::{
214 alloc::{box_ext::BoxExt, AllocError, Flags
215 error::{self, Error},
216 sync::Arc,
217 sync::UniqueArc,
218 types::{Opaque, ScopeGuard},
219 };
220 use alloc::boxed::Box;
221 use core::{
222 cell::UnsafeCell,
223 convert::Infallible,
224 marker::PhantomData,
225 mem::MaybeUninit,
226 num::*,
227 pin::Pin,
228 ptr::{self, NonNull},
229 };
230
231 #[doc(hidden)]
232 pub mod __internal;
233 #[doc(hidden)]
234 pub mod macros;
235
236 /// Initialize and pin a type directly on the
237 ///
238 /// # Examples
239 ///
240 /// ```rust
241 /// # #![allow(clippy::disallowed_names)]
242 /// # use kernel::{init, macros::pin_data, pin
243 /// # use core::pin::Pin;
244 /// #[pin_data]
245 /// struct Foo {
246 /// #[pin]
247 /// a: Mutex<usize>,
248 /// b: Bar,
249 /// }
250 ///
251 /// #[pin_data]
252 /// struct Bar {
253 /// x: u32,
254 /// }
255 ///
256 /// stack_pin_init!(let foo = pin_init!(Foo {
257 /// a <- new_mutex!(42),
258 /// b: Bar {
259 /// x: 64,
260 /// },
261 /// }));
262 /// let foo: Pin<&mut Foo> = foo;
263 /// pr_info!("a: {}", &*foo.a.lock());
264 /// ```
265 ///
266 /// # Syntax
267 ///
268 /// A normal `let` binding with optional type
269 /// [`PinInit`]/[`Init`] with the error type [
270 /// type, then use [`stack_try_pin_init!`].
271 ///
272 /// [`stack_try_pin_init!`]: crate::stack_try_
273 #[macro_export]
274 macro_rules! stack_pin_init {
275 (let $var:ident $(: $t:ty)? = $val:expr) =
276 let val = $val;
277 let mut $var = ::core::pin::pin!($crat
278 let mut $var = match $crate::init::__i
279 Ok(res) => res,
280 Err(x) => {
281 let x: ::core::convert::Infall
282 match x {}
283 }
284 };
285 };
286 }
287
288 /// Initialize and pin a type directly on the
289 ///
290 /// # Examples
291 ///
292 /// ```rust,ignore
293 /// # #![allow(clippy::disallowed_names)]
294 /// # use kernel::{init, pin_init, stack_try_p
295 /// # use macros::pin_data;
296 /// # use core::{alloc::AllocError, pin::Pin};
297 /// #[pin_data]
298 /// struct Foo {
299 /// #[pin]
300 /// a: Mutex<usize>,
301 /// b: Box<Bar>,
302 /// }
303 ///
304 /// struct Bar {
305 /// x: u32,
306 /// }
307 ///
308 /// stack_try_pin_init!(let foo: Result<Pin<&m
309 /// a <- new_mutex!(42),
310 /// b: Box::new(Bar {
311 /// x: 64,
312 /// }, GFP_KERNEL)?,
313 /// }));
314 /// let foo = foo.unwrap();
315 /// pr_info!("a: {}", &*foo.a.lock());
316 /// ```
317 ///
318 /// ```rust,ignore
319 /// # #![allow(clippy::disallowed_names)]
320 /// # use kernel::{init, pin_init, stack_try_p
321 /// # use macros::pin_data;
322 /// # use core::{alloc::AllocError, pin::Pin};
323 /// #[pin_data]
324 /// struct Foo {
325 /// #[pin]
326 /// a: Mutex<usize>,
327 /// b: Box<Bar>,
328 /// }
329 ///
330 /// struct Bar {
331 /// x: u32,
332 /// }
333 ///
334 /// stack_try_pin_init!(let foo: Pin<&mut Foo>
335 /// a <- new_mutex!(42),
336 /// b: Box::new(Bar {
337 /// x: 64,
338 /// }, GFP_KERNEL)?,
339 /// }));
340 /// pr_info!("a: {}", &*foo.a.lock());
341 /// # Ok::<_, AllocError>(())
342 /// ```
343 ///
344 /// # Syntax
345 ///
346 /// A normal `let` binding with optional type
347 /// [`PinInit`]/[`Init`]. This macro assigns a
348 /// `=` will propagate this error.
349 #[macro_export]
350 macro_rules! stack_try_pin_init {
351 (let $var:ident $(: $t:ty)? = $val:expr) =
352 let val = $val;
353 let mut $var = ::core::pin::pin!($crat
354 let mut $var = $crate::init::__interna
355 };
356 (let $var:ident $(: $t:ty)? =? $val:expr)
357 let val = $val;
358 let mut $var = ::core::pin::pin!($crat
359 let mut $var = $crate::init::__interna
360 };
361 }
362
363 /// Construct an in-place, pinned initializer
364 ///
365 /// This macro defaults the error to [`Infalli
366 /// [`try_pin_init!`].
367 ///
368 /// The syntax is almost identical to that of
369 ///
370 /// ```rust
371 /// # #![allow(clippy::disallowed_names)]
372 /// # use kernel::{init, pin_init, macros::pin
373 /// # use core::pin::Pin;
374 /// #[pin_data]
375 /// struct Foo {
376 /// a: usize,
377 /// b: Bar,
378 /// }
379 ///
380 /// #[pin_data]
381 /// struct Bar {
382 /// x: u32,
383 /// }
384 ///
385 /// # fn demo() -> impl PinInit<Foo> {
386 /// let a = 42;
387 ///
388 /// let initializer = pin_init!(Foo {
389 /// a,
390 /// b: Bar {
391 /// x: 64,
392 /// },
393 /// });
394 /// # initializer }
395 /// # Box::pin_init(demo(), GFP_KERNEL).unwrap
396 /// ```
397 ///
398 /// Arbitrary Rust expressions can be used to
399 ///
400 /// The fields are initialized in the order th
401 /// to read already initialized fields using r
402 ///
403 /// IMPORTANT: You are not allowed to create r
404 /// initializer.
405 ///
406 /// # Init-functions
407 ///
408 /// When working with this API it is often des
409 /// giving access to all fields. This is where
410 /// that would return a new instance of your t
411 /// However, there are a few extra things to k
412 ///
413 /// To create an initializer function, simply
414 ///
415 /// ```rust
416 /// # #![allow(clippy::disallowed_names)]
417 /// # use kernel::{init, pin_init, init::*};
418 /// # use core::pin::Pin;
419 /// # #[pin_data]
420 /// # struct Foo {
421 /// # a: usize,
422 /// # b: Bar,
423 /// # }
424 /// # #[pin_data]
425 /// # struct Bar {
426 /// # x: u32,
427 /// # }
428 /// impl Foo {
429 /// fn new() -> impl PinInit<Self> {
430 /// pin_init!(Self {
431 /// a: 42,
432 /// b: Bar {
433 /// x: 64,
434 /// },
435 /// })
436 /// }
437 /// }
438 /// ```
439 ///
440 /// Users of `Foo` can now create it like this
441 ///
442 /// ```rust
443 /// # #![allow(clippy::disallowed_names)]
444 /// # use kernel::{init, pin_init, macros::pin
445 /// # use core::pin::Pin;
446 /// # #[pin_data]
447 /// # struct Foo {
448 /// # a: usize,
449 /// # b: Bar,
450 /// # }
451 /// # #[pin_data]
452 /// # struct Bar {
453 /// # x: u32,
454 /// # }
455 /// # impl Foo {
456 /// # fn new() -> impl PinInit<Self> {
457 /// # pin_init!(Self {
458 /// # a: 42,
459 /// # b: Bar {
460 /// # x: 64,
461 /// # },
462 /// # })
463 /// # }
464 /// # }
465 /// let foo = Box::pin_init(Foo::new(), GFP_KE
466 /// ```
467 ///
468 /// They can also easily embed it into their o
469 ///
470 /// ```rust
471 /// # #![allow(clippy::disallowed_names)]
472 /// # use kernel::{init, pin_init, macros::pin
473 /// # use core::pin::Pin;
474 /// # #[pin_data]
475 /// # struct Foo {
476 /// # a: usize,
477 /// # b: Bar,
478 /// # }
479 /// # #[pin_data]
480 /// # struct Bar {
481 /// # x: u32,
482 /// # }
483 /// # impl Foo {
484 /// # fn new() -> impl PinInit<Self> {
485 /// # pin_init!(Self {
486 /// # a: 42,
487 /// # b: Bar {
488 /// # x: 64,
489 /// # },
490 /// # })
491 /// # }
492 /// # }
493 /// #[pin_data]
494 /// struct FooContainer {
495 /// #[pin]
496 /// foo1: Foo,
497 /// #[pin]
498 /// foo2: Foo,
499 /// other: u32,
500 /// }
501 ///
502 /// impl FooContainer {
503 /// fn new(other: u32) -> impl PinInit<Sel
504 /// pin_init!(Self {
505 /// foo1 <- Foo::new(),
506 /// foo2 <- Foo::new(),
507 /// other,
508 /// })
509 /// }
510 /// }
511 /// ```
512 ///
513 /// Here we see that when using `pin_init!` wi
514 /// This signifies that the given field is ini
515 /// writing the field (in this case `other`) w
516 ///
517 /// # Syntax
518 ///
519 /// As already mentioned in the examples above
520 /// the following modifications is expected:
521 /// - Fields that you want to initialize in-pl
522 /// - In front of the initializer you can writ
523 /// pointer named `this` inside of the initi
524 /// - Using struct update syntax one can place
525 /// struct, this initializes every field wit
526 /// body. This can only be done if [`Zeroabl
527 ///
528 /// For instance:
529 ///
530 /// ```rust
531 /// # use kernel::{macros::{Zeroable, pin_data
532 /// # use core::{ptr::addr_of_mut, marker::Pha
533 /// #[pin_data]
534 /// #[derive(Zeroable)]
535 /// struct Buf {
536 /// // `ptr` points into `buf`.
537 /// ptr: *mut u8,
538 /// buf: [u8; 64],
539 /// #[pin]
540 /// pin: PhantomPinned,
541 /// }
542 /// pin_init!(&this in Buf {
543 /// buf: [0; 64],
544 /// ptr: unsafe { addr_of_mut!((*this.as_p
545 /// pin: PhantomPinned,
546 /// });
547 /// pin_init!(Buf {
548 /// buf: [1; 64],
549 /// ..Zeroable::zeroed()
550 /// });
551 /// ```
552 ///
553 /// [`try_pin_init!`]: kernel::try_pin_init
554 /// [`NonNull<Self>`]: core::ptr::NonNull
555 // For a detailed example of how this macro wo
556 // module `__internal` inside of `init/__inter
557 #[macro_export]
558 macro_rules! pin_init {
559 ($(&$this:ident in)? $t:ident $(::<$($gene
560 $($fields:tt)*
561 }) => {
562 $crate::__init_internal!(
563 @this($($this)?),
564 @typ($t $(::<$($generics),*>)?),
565 @fields($($fields)*),
566 @error(::core::convert::Infallible
567 @data(PinData, use_data),
568 @has_data(HasPinData, __pin_data),
569 @construct_closure(pin_init_from_c
570 @munch_fields($($fields)*),
571 )
572 };
573 }
574
575 /// Construct an in-place, fallible pinned ini
576 ///
577 /// If the initialization can complete without
578 ///
579 /// You can use the `?` operator or use `retur
580 /// initialization and return the error.
581 ///
582 /// IMPORTANT: if you have `unsafe` code insid
583 /// initialization fails, the memory can be sa
584 ///
585 /// This macro defaults the error to [`Error`]
586 ///
587 /// The syntax is identical to [`pin_init!`] w
588 /// after the `struct` initializer to specify
589 ///
590 /// # Examples
591 ///
592 /// ```rust
593 /// # #![feature(new_uninit)]
594 /// use kernel::{init::{self, PinInit}, error:
595 /// #[pin_data]
596 /// struct BigBuf {
597 /// big: Box<[u8; 1024 * 1024 * 1024]>,
598 /// small: [u8; 1024 * 1024],
599 /// ptr: *mut u8,
600 /// }
601 ///
602 /// impl BigBuf {
603 /// fn new() -> impl PinInit<Self, Error>
604 /// try_pin_init!(Self {
605 /// big: Box::init(init::zeroed(),
606 /// small: [0; 1024 * 1024],
607 /// ptr: core::ptr::null_mut(),
608 /// }? Error)
609 /// }
610 /// }
611 /// ```
612 // For a detailed example of how this macro wo
613 // module `__internal` inside of `init/__inter
614 #[macro_export]
615 macro_rules! try_pin_init {
616 ($(&$this:ident in)? $t:ident $(::<$($gene
617 $($fields:tt)*
618 }) => {
619 $crate::__init_internal!(
620 @this($($this)?),
621 @typ($t $(::<$($generics),*>)? ),
622 @fields($($fields)*),
623 @error($crate::error::Error),
624 @data(PinData, use_data),
625 @has_data(HasPinData, __pin_data),
626 @construct_closure(pin_init_from_c
627 @munch_fields($($fields)*),
628 )
629 };
630 ($(&$this:ident in)? $t:ident $(::<$($gene
631 $($fields:tt)*
632 }? $err:ty) => {
633 $crate::__init_internal!(
634 @this($($this)?),
635 @typ($t $(::<$($generics),*>)? ),
636 @fields($($fields)*),
637 @error($err),
638 @data(PinData, use_data),
639 @has_data(HasPinData, __pin_data),
640 @construct_closure(pin_init_from_c
641 @munch_fields($($fields)*),
642 )
643 };
644 }
645
646 /// Construct an in-place initializer for `str
647 ///
648 /// This macro defaults the error to [`Infalli
649 /// [`try_init!`].
650 ///
651 /// The syntax is identical to [`pin_init!`] a
652 /// - `unsafe` code must guarantee either full
653 /// deallocation of the memory.
654 /// - the fields are initialized in the order
655 /// - no references to fields are allowed to b
656 ///
657 /// This initializer is for initializing data
658 /// pin-initialize, use [`pin_init!`].
659 ///
660 /// [`try_init!`]: crate::try_init!
661 // For a detailed example of how this macro wo
662 // module `__internal` inside of `init/__inter
663 #[macro_export]
664 macro_rules! init {
665 ($(&$this:ident in)? $t:ident $(::<$($gene
666 $($fields:tt)*
667 }) => {
668 $crate::__init_internal!(
669 @this($($this)?),
670 @typ($t $(::<$($generics),*>)?),
671 @fields($($fields)*),
672 @error(::core::convert::Infallible
673 @data(InitData, /*no use_data*/),
674 @has_data(HasInitData, __init_data
675 @construct_closure(init_from_closu
676 @munch_fields($($fields)*),
677 )
678 }
679 }
680
681 /// Construct an in-place fallible initializer
682 ///
683 /// This macro defaults the error to [`Error`]
684 /// [`init!`].
685 ///
686 /// The syntax is identical to [`try_pin_init!
687 /// append `? $type` after the `struct` initia
688 /// The safety caveats from [`try_pin_init!`]
689 /// - `unsafe` code must guarantee either full
690 /// deallocation of the memory.
691 /// - the fields are initialized in the order
692 /// - no references to fields are allowed to b
693 ///
694 /// # Examples
695 ///
696 /// ```rust
697 /// use kernel::{init::{PinInit, zeroed}, erro
698 /// struct BigBuf {
699 /// big: Box<[u8; 1024 * 1024 * 1024]>,
700 /// small: [u8; 1024 * 1024],
701 /// }
702 ///
703 /// impl BigBuf {
704 /// fn new() -> impl Init<Self, Error> {
705 /// try_init!(Self {
706 /// big: Box::init(zeroed(), GFP_K
707 /// small: [0; 1024 * 1024],
708 /// }? Error)
709 /// }
710 /// }
711 /// ```
712 // For a detailed example of how this macro wo
713 // module `__internal` inside of `init/__inter
714 #[macro_export]
715 macro_rules! try_init {
716 ($(&$this:ident in)? $t:ident $(::<$($gene
717 $($fields:tt)*
718 }) => {
719 $crate::__init_internal!(
720 @this($($this)?),
721 @typ($t $(::<$($generics),*>)?),
722 @fields($($fields)*),
723 @error($crate::error::Error),
724 @data(InitData, /*no use_data*/),
725 @has_data(HasInitData, __init_data
726 @construct_closure(init_from_closu
727 @munch_fields($($fields)*),
728 )
729 };
730 ($(&$this:ident in)? $t:ident $(::<$($gene
731 $($fields:tt)*
732 }? $err:ty) => {
733 $crate::__init_internal!(
734 @this($($this)?),
735 @typ($t $(::<$($generics),*>)?),
736 @fields($($fields)*),
737 @error($err),
738 @data(InitData, /*no use_data*/),
739 @has_data(HasInitData, __init_data
740 @construct_closure(init_from_closu
741 @munch_fields($($fields)*),
742 )
743 };
744 }
745
746 /// Asserts that a field on a struct using `#[
747 /// structurally pinned.
748 ///
749 /// # Example
750 ///
751 /// This will succeed:
752 /// ```
753 /// use kernel::assert_pinned;
754 /// #[pin_data]
755 /// struct MyStruct {
756 /// #[pin]
757 /// some_field: u64,
758 /// }
759 ///
760 /// assert_pinned!(MyStruct, some_field, u64);
761 /// ```
762 ///
763 /// This will fail:
764 // TODO: replace with `compile_fail` when supp
765 /// ```ignore
766 /// use kernel::assert_pinned;
767 /// #[pin_data]
768 /// struct MyStruct {
769 /// some_field: u64,
770 /// }
771 ///
772 /// assert_pinned!(MyStruct, some_field, u64);
773 /// ```
774 ///
775 /// Some uses of the macro may trigger the `ca
776 /// work around this, you may pass the `inline
777 /// only be used when the macro is invoked fro
778 /// ```
779 /// use kernel::assert_pinned;
780 /// #[pin_data]
781 /// struct Foo<T> {
782 /// #[pin]
783 /// elem: T,
784 /// }
785 ///
786 /// impl<T> Foo<T> {
787 /// fn project(self: Pin<&mut Self>) -> Pi
788 /// assert_pinned!(Foo<T>, elem, T, in
789 ///
790 /// // SAFETY: The field is structural
791 /// unsafe { self.map_unchecked_mut(|m
792 /// }
793 /// }
794 /// ```
795 #[macro_export]
796 macro_rules! assert_pinned {
797 ($ty:ty, $field:ident, $field_ty:ty, inlin
798 let _ = move |ptr: *mut $field_ty| {
799 // SAFETY: This code is unreachabl
800 let data = unsafe { <$ty as $crate
801 let init = $crate::init::__interna
802 // SAFETY: This code is unreachabl
803 unsafe { data.$field(ptr, init) }.
804 };
805 };
806
807 ($ty:ty, $field:ident, $field_ty:ty) => {
808 const _: () = {
809 $crate::assert_pinned!($ty, $field
810 };
811 };
812 }
813
814 /// A pin-initializer for the type `T`.
815 ///
816 /// To use this initializer, you will need a s
817 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`
818 /// [`InPlaceInit::pin_init`] function of a sm
819 ///
820 /// Also see the [module description](self).
821 ///
822 /// # Safety
823 ///
824 /// When implementing this trait you will need
825 /// cases where a manual implementation is nec
826 ///
827 /// The [`PinInit::__pinned_init`] function:
828 /// - returns `Ok(())` if it initialized every
829 /// - returns `Err(err)` if it encountered an
830 /// - `slot` can be deallocated without UB
831 /// - `slot` does not need to be dropped,
832 /// - `slot` is not partially initialized.
833 /// - while constructing the `T` at `slot` it
834 ///
835 /// [`Arc<T>`]: crate::sync::Arc
836 /// [`Arc::pin_init`]: crate::sync::Arc::pin_i
837 #[must_use = "An initializer must be used in o
838 pub unsafe trait PinInit<T: ?Sized, E = Infall
839 /// Initializes `slot`.
840 ///
841 /// # Safety
842 ///
843 /// - `slot` is a valid pointer to uniniti
844 /// - the caller does not touch `slot` whe
845 /// deallocate.
846 /// - `slot` will not move until it is dro
847 unsafe fn __pinned_init(self, slot: *mut T
848
849 /// First initializes the value using `sel
850 /// value.
851 ///
852 /// If `f` returns an error the value is d
853 ///
854 /// # Examples
855 ///
856 /// ```rust
857 /// # #![allow(clippy::disallowed_names)]
858 /// use kernel::{types::Opaque, init::pin_
859 /// #[repr(C)]
860 /// struct RawFoo([u8; 16]);
861 /// extern {
862 /// fn init_foo(_: *mut RawFoo);
863 /// }
864 ///
865 /// #[pin_data]
866 /// struct Foo {
867 /// #[pin]
868 /// raw: Opaque<RawFoo>,
869 /// }
870 ///
871 /// impl Foo {
872 /// fn setup(self: Pin<&mut Self>) {
873 /// pr_info!("Setting up foo");
874 /// }
875 /// }
876 ///
877 /// let foo = pin_init!(Foo {
878 /// raw <- unsafe {
879 /// Opaque::ffi_init(|s| {
880 /// init_foo(s);
881 /// })
882 /// },
883 /// }).pin_chain(|foo| {
884 /// foo.setup();
885 /// Ok(())
886 /// });
887 /// ```
888 fn pin_chain<F>(self, f: F) -> ChainPinIni
889 where
890 F: FnOnce(Pin<&mut T>) -> Result<(), E
891 {
892 ChainPinInit(self, f, PhantomData)
893 }
894 }
895
896 /// An initializer returned by [`PinInit::pin_
897 pub struct ChainPinInit<I, F, T: ?Sized, E>(I,
898
899 // SAFETY: The `__pinned_init` function is imp
900 // - returns `Ok(())` on successful initializa
901 // - returns `Err(err)` on error and in this c
902 // - considers `slot` pinned.
903 unsafe impl<T: ?Sized, E, I, F> PinInit<T, E>
904 where
905 I: PinInit<T, E>,
906 F: FnOnce(Pin<&mut T>) -> Result<(), E>,
907 {
908 unsafe fn __pinned_init(self, slot: *mut T
909 // SAFETY: All requirements fulfilled
910 unsafe { self.0.__pinned_init(slot)? }
911 // SAFETY: The above call initialized
912 let val = unsafe { &mut *slot };
913 // SAFETY: `slot` is considered pinned
914 let val = unsafe { Pin::new_unchecked(
915 // SAFETY: `slot` was initialized abov
916 (self.1)(val).inspect_err(|_| unsafe {
917 }
918 }
919
920 /// An initializer for `T`.
921 ///
922 /// To use this initializer, you will need a s
923 /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`
924 /// [`InPlaceInit::init`] function of a smart
925 /// [`PinInit<T, E>`] is a super trait, you ca
926 ///
927 /// Also see the [module description](self).
928 ///
929 /// # Safety
930 ///
931 /// When implementing this trait you will need
932 /// cases where a manual implementation is nec
933 ///
934 /// The [`Init::__init`] function:
935 /// - returns `Ok(())` if it initialized every
936 /// - returns `Err(err)` if it encountered an
937 /// - `slot` can be deallocated without UB
938 /// - `slot` does not need to be dropped,
939 /// - `slot` is not partially initialized.
940 /// - while constructing the `T` at `slot` it
941 ///
942 /// The `__pinned_init` function from the supe
943 /// code as `__init`.
944 ///
945 /// Contrary to its supertype [`PinInit<T, E>`
946 /// move the pointee after initialization.
947 ///
948 /// [`Arc<T>`]: crate::sync::Arc
949 #[must_use = "An initializer must be used in o
950 pub unsafe trait Init<T: ?Sized, E = Infallibl
951 /// Initializes `slot`.
952 ///
953 /// # Safety
954 ///
955 /// - `slot` is a valid pointer to uniniti
956 /// - the caller does not touch `slot` whe
957 /// deallocate.
958 unsafe fn __init(self, slot: *mut T) -> Re
959
960 /// First initializes the value using `sel
961 /// value.
962 ///
963 /// If `f` returns an error the value is d
964 ///
965 /// # Examples
966 ///
967 /// ```rust
968 /// # #![allow(clippy::disallowed_names)]
969 /// use kernel::{types::Opaque, init::{sel
970 /// struct Foo {
971 /// buf: [u8; 1_000_000],
972 /// }
973 ///
974 /// impl Foo {
975 /// fn setup(&mut self) {
976 /// pr_info!("Setting up foo");
977 /// }
978 /// }
979 ///
980 /// let foo = init!(Foo {
981 /// buf <- init::zeroed()
982 /// }).chain(|foo| {
983 /// foo.setup();
984 /// Ok(())
985 /// });
986 /// ```
987 fn chain<F>(self, f: F) -> ChainInit<Self,
988 where
989 F: FnOnce(&mut T) -> Result<(), E>,
990 {
991 ChainInit(self, f, PhantomData)
992 }
993 }
994
995 /// An initializer returned by [`Init::chain`]
996 pub struct ChainInit<I, F, T: ?Sized, E>(I, F,
997
998 // SAFETY: The `__init` function is implemente
999 // - returns `Ok(())` on successful initializa
1000 // - returns `Err(err)` on error and in this
1001 unsafe impl<T: ?Sized, E, I, F> Init<T, E> fo
1002 where
1003 I: Init<T, E>,
1004 F: FnOnce(&mut T) -> Result<(), E>,
1005 {
1006 unsafe fn __init(self, slot: *mut T) -> R
1007 // SAFETY: All requirements fulfilled
1008 unsafe { self.0.__pinned_init(slot)?
1009 // SAFETY: The above call initialized
1010 (self.1)(unsafe { &mut *slot }).inspe
1011 // SAFETY: `slot` was initialized
1012 unsafe { core::ptr::drop_in_place
1013 }
1014 }
1015
1016 // SAFETY: `__pinned_init` behaves exactly th
1017 unsafe impl<T: ?Sized, E, I, F> PinInit<T, E>
1018 where
1019 I: Init<T, E>,
1020 F: FnOnce(&mut T) -> Result<(), E>,
1021 {
1022 unsafe fn __pinned_init(self, slot: *mut
1023 // SAFETY: `__init` has less strict r
1024 unsafe { self.__init(slot) }
1025 }
1026 }
1027
1028 /// Creates a new [`PinInit<T, E>`] from the
1029 ///
1030 /// # Safety
1031 ///
1032 /// The closure:
1033 /// - returns `Ok(())` if it initialized ever
1034 /// - returns `Err(err)` if it encountered an
1035 /// - `slot` can be deallocated without U
1036 /// - `slot` does not need to be dropped,
1037 /// - `slot` is not partially initialized
1038 /// - may assume that the `slot` does not mov
1039 /// - while constructing the `T` at `slot` it
1040 #[inline]
1041 pub const unsafe fn pin_init_from_closure<T:
1042 f: impl FnOnce(*mut T) -> Result<(), E>,
1043 ) -> impl PinInit<T, E> {
1044 __internal::InitClosure(f, PhantomData)
1045 }
1046
1047 /// Creates a new [`Init<T, E>`] from the giv
1048 ///
1049 /// # Safety
1050 ///
1051 /// The closure:
1052 /// - returns `Ok(())` if it initialized ever
1053 /// - returns `Err(err)` if it encountered an
1054 /// - `slot` can be deallocated without U
1055 /// - `slot` does not need to be dropped,
1056 /// - `slot` is not partially initialized
1057 /// - the `slot` may move after initializatio
1058 /// - while constructing the `T` at `slot` it
1059 #[inline]
1060 pub const unsafe fn init_from_closure<T: ?Siz
1061 f: impl FnOnce(*mut T) -> Result<(), E>,
1062 ) -> impl Init<T, E> {
1063 __internal::InitClosure(f, PhantomData)
1064 }
1065
1066 /// An initializer that leaves the memory uni
1067 ///
1068 /// The initializer is a no-op. The `slot` me
1069 #[inline]
1070 pub fn uninit<T, E>() -> impl Init<MaybeUnini
1071 // SAFETY: The memory is allowed to be un
1072 unsafe { init_from_closure(|_| Ok(())) }
1073 }
1074
1075 /// Initializes an array by initializing each
1076 ///
1077 /// # Examples
1078 ///
1079 /// ```rust
1080 /// use kernel::{error::Error, init::init_arr
1081 /// let array: Box<[usize; 1_000]> = Box::ini
1082 /// assert_eq!(array.len(), 1_000);
1083 /// ```
1084 pub fn init_array_from_fn<I, const N: usize,
1085 mut make_init: impl FnMut(usize) -> I,
1086 ) -> impl Init<[T; N], E>
1087 where
1088 I: Init<T, E>,
1089 {
1090 let init = move |slot: *mut [T; N]| {
1091 let slot = slot.cast::<T>();
1092 // Counts the number of initialized e
1093 // `slot`.
1094 let mut init_count = ScopeGuard::new_
1095 // We now free every element that
1096 // SAFETY: The loop initialized e
1097 // return `Err` below, the caller
1098 // uninitialized.
1099 unsafe { ptr::drop_in_place(ptr::
1100 });
1101 for i in 0..N {
1102 let init = make_init(i);
1103 // SAFETY: Since 0 <= `i` < N, it
1104 let ptr = unsafe { slot.add(i) };
1105 // SAFETY: The pointer is derived
1106 // requirements.
1107 unsafe { init.__init(ptr) }?;
1108 *init_count += 1;
1109 }
1110 init_count.dismiss();
1111 Ok(())
1112 };
1113 // SAFETY: The initializer above initiali
1114 // any initialized elements and returns `
1115 unsafe { init_from_closure(init) }
1116 }
1117
1118 /// Initializes an array by initializing each
1119 ///
1120 /// # Examples
1121 ///
1122 /// ```rust
1123 /// use kernel::{sync::{Arc, Mutex}, init::pi
1124 /// let array: Arc<[Mutex<usize>; 1_000]> =
1125 /// Arc::pin_init(pin_init_array_from_fn(
1126 /// assert_eq!(array.len(), 1_000);
1127 /// ```
1128 pub fn pin_init_array_from_fn<I, const N: usi
1129 mut make_init: impl FnMut(usize) -> I,
1130 ) -> impl PinInit<[T; N], E>
1131 where
1132 I: PinInit<T, E>,
1133 {
1134 let init = move |slot: *mut [T; N]| {
1135 let slot = slot.cast::<T>();
1136 // Counts the number of initialized e
1137 // `slot`.
1138 let mut init_count = ScopeGuard::new_
1139 // We now free every element that
1140 // SAFETY: The loop initialized e
1141 // return `Err` below, the caller
1142 // uninitialized.
1143 unsafe { ptr::drop_in_place(ptr::
1144 });
1145 for i in 0..N {
1146 let init = make_init(i);
1147 // SAFETY: Since 0 <= `i` < N, it
1148 let ptr = unsafe { slot.add(i) };
1149 // SAFETY: The pointer is derived
1150 // requirements.
1151 unsafe { init.__pinned_init(ptr)
1152 *init_count += 1;
1153 }
1154 init_count.dismiss();
1155 Ok(())
1156 };
1157 // SAFETY: The initializer above initiali
1158 // any initialized elements and returns `
1159 unsafe { pin_init_from_closure(init) }
1160 }
1161
1162 // SAFETY: Every type can be initialized by-v
1163 unsafe impl<T, E> Init<T, E> for T {
1164 unsafe fn __init(self, slot: *mut T) -> R
1165 unsafe { slot.write(self) };
1166 Ok(())
1167 }
1168 }
1169
1170 // SAFETY: Every type can be initialized by-v
1171 unsafe impl<T, E> PinInit<T, E> for T {
1172 unsafe fn __pinned_init(self, slot: *mut
1173 unsafe { self.__init(slot) }
1174 }
1175 }
1176
1177 /// Smart pointer that can initialize memory
1178 pub trait InPlaceInit<T>: Sized {
1179 /// Pinned version of `Self`.
1180 ///
1181 /// If a type already implicitly pins its
1182 /// `Self`, otherwise just use `Pin<Self>
1183 type PinnedSelf;
1184
1185 /// Use the given pin-initializer to pin-
1186 /// type.
1187 ///
1188 /// If `T: !Unpin` it will not be able to
1189 fn try_pin_init<E>(init: impl PinInit<T,
1190 where
1191 E: From<AllocError>;
1192
1193 /// Use the given pin-initializer to pin-
1194 /// type.
1195 ///
1196 /// If `T: !Unpin` it will not be able to
1197 fn pin_init<E>(init: impl PinInit<T, E>,
1198 where
1199 Error: From<E>,
1200 {
1201 // SAFETY: We delegate to `init` and
1202 let init = unsafe {
1203 pin_init_from_closure(|slot| init
1204 };
1205 Self::try_pin_init(init, flags)
1206 }
1207
1208 /// Use the given initializer to in-place
1209 fn try_init<E>(init: impl Init<T, E>, fla
1210 where
1211 E: From<AllocError>;
1212
1213 /// Use the given initializer to in-place
1214 fn init<E>(init: impl Init<T, E>, flags:
1215 where
1216 Error: From<E>,
1217 {
1218 // SAFETY: We delegate to `init` and
1219 let init = unsafe {
1220 init_from_closure(|slot| init.__p
1221 };
1222 Self::try_init(init, flags)
1223 }
1224 }
1225
1226 impl<T> InPlaceInit<T> for Arc<T> {
1227 type PinnedSelf = Self;
1228
1229 #[inline]
1230 fn try_pin_init<E>(init: impl PinInit<T,
1231 where
1232 E: From<AllocError>,
1233 {
1234 UniqueArc::try_pin_init(init, flags).
1235 }
1236
1237 #[inline]
1238 fn try_init<E>(init: impl Init<T, E>, fla
1239 where
1240 E: From<AllocError>,
1241 {
1242 UniqueArc::try_init(init, flags).map(
1243 }
1244 }
1245
1246 impl<T> InPlaceInit<T> for Box<T> {
1247 type PinnedSelf = Pin<Self>;
1248
1249 #[inline]
1250 fn try_pin_init<E>(init: impl PinInit<T,
1251 where
1252 E: From<AllocError>,
1253 {
1254 <Box<_> as BoxExt<_>>::new_uninit(fla
1255 }
1256
1257 #[inline]
1258 fn try_init<E>(init: impl Init<T, E>, fla
1259 where
1260 E: From<AllocError>,
1261 {
1262 <Box<_> as BoxExt<_>>::new_uninit(fla
1263 }
1264 }
1265
1266 impl<T> InPlaceInit<T> for UniqueArc<T> {
1267 type PinnedSelf = Pin<Self>;
1268
1269 #[inline]
1270 fn try_pin_init<E>(init: impl PinInit<T,
1271 where
1272 E: From<AllocError>,
1273 {
1274 UniqueArc::new_uninit(flags)?.write_p
1275 }
1276
1277 #[inline]
1278 fn try_init<E>(init: impl Init<T, E>, fla
1279 where
1280 E: From<AllocError>,
1281 {
1282 UniqueArc::new_uninit(flags)?.write_i
1283 }
1284 }
1285
1286 /// Smart pointer containing uninitialized me
1287 pub trait InPlaceWrite<T> {
1288 /// The type `Self` turns into when the c
1289 type Initialized;
1290
1291 /// Use the given initializer to write a
1292 ///
1293 /// Does not drop the current value and c
1294 fn write_init<E>(self, init: impl Init<T,
1295
1296 /// Use the given pin-initializer to writ
1297 ///
1298 /// Does not drop the current value and c
1299 fn write_pin_init<E>(self, init: impl Pin
1300 }
1301
1302 impl<T> InPlaceWrite<T> for Box<MaybeUninit<T
1303 type Initialized = Box<T>;
1304
1305 fn write_init<E>(mut self, init: impl Ini
1306 let slot = self.as_mut_ptr();
1307 // SAFETY: When init errors/panics, s
1308 // slot is valid.
1309 unsafe { init.__init(slot)? };
1310 // SAFETY: All fields have been initi
1311 Ok(unsafe { self.assume_init() })
1312 }
1313
1314 fn write_pin_init<E>(mut self, init: impl
1315 let slot = self.as_mut_ptr();
1316 // SAFETY: When init errors/panics, s
1317 // slot is valid and will not be move
1318 unsafe { init.__pinned_init(slot)? };
1319 // SAFETY: All fields have been initi
1320 Ok(unsafe { self.assume_init() }.into
1321 }
1322 }
1323
1324 impl<T> InPlaceWrite<T> for UniqueArc<MaybeUn
1325 type Initialized = UniqueArc<T>;
1326
1327 fn write_init<E>(mut self, init: impl Ini
1328 let slot = self.as_mut_ptr();
1329 // SAFETY: When init errors/panics, s
1330 // slot is valid.
1331 unsafe { init.__init(slot)? };
1332 // SAFETY: All fields have been initi
1333 Ok(unsafe { self.assume_init() })
1334 }
1335
1336 fn write_pin_init<E>(mut self, init: impl
1337 let slot = self.as_mut_ptr();
1338 // SAFETY: When init errors/panics, s
1339 // slot is valid and will not be move
1340 unsafe { init.__pinned_init(slot)? };
1341 // SAFETY: All fields have been initi
1342 Ok(unsafe { self.assume_init() }.into
1343 }
1344 }
1345
1346 /// Trait facilitating pinned destruction.
1347 ///
1348 /// Use [`pinned_drop`] to implement this tra
1349 ///
1350 /// ```rust
1351 /// # use kernel::sync::Mutex;
1352 /// use kernel::macros::pinned_drop;
1353 /// use core::pin::Pin;
1354 /// #[pin_data(PinnedDrop)]
1355 /// struct Foo {
1356 /// #[pin]
1357 /// mtx: Mutex<usize>,
1358 /// }
1359 ///
1360 /// #[pinned_drop]
1361 /// impl PinnedDrop for Foo {
1362 /// fn drop(self: Pin<&mut Self>) {
1363 /// pr_info!("Foo is being dropped!")
1364 /// }
1365 /// }
1366 /// ```
1367 ///
1368 /// # Safety
1369 ///
1370 /// This trait must be implemented via the [`
1371 ///
1372 /// [`pinned_drop`]: kernel::macros::pinned_d
1373 pub unsafe trait PinnedDrop: __internal::HasP
1374 /// Executes the pinned destructor of thi
1375 ///
1376 /// While this function is marked safe, i
1377 /// reason it takes an additional paramet
1378 /// and thus prevents this function from
1379 ///
1380 /// This extra parameter will be generate
1381 /// automatically.
1382 fn drop(self: Pin<&mut Self>, only_call_f
1383 }
1384
1385 /// Marker trait for types that can be initia
1386 ///
1387 /// # Safety
1388 ///
1389 /// The bit pattern consisting of only zeroes
1390 /// this is not UB:
1391 ///
1392 /// ```rust,ignore
1393 /// let val: Self = unsafe { core::mem::zeroe
1394 /// ```
1395 pub unsafe trait Zeroable {}
1396
1397 /// Create a new zeroed T.
1398 ///
1399 /// The returned initializer will write `0x00
1400 #[inline]
1401 pub fn zeroed<T: Zeroable>() -> impl Init<T>
1402 // SAFETY: Because `T: Zeroable`, all byt
1403 // and because we write all zeroes, the m
1404 unsafe {
1405 init_from_closure(|slot: *mut T| {
1406 slot.write_bytes(0, 1);
1407 Ok(())
1408 })
1409 }
1410 }
1411
1412 macro_rules! impl_zeroable {
1413 ($($({$($generics:tt)*})? $t:ty, )*) => {
1414 $(unsafe impl$($($generics)*)? Zeroab
1415 };
1416 }
1417
1418 impl_zeroable! {
1419 // SAFETY: All primitives that are allowe
1420 bool,
1421 char,
1422 u8, u16, u32, u64, u128, usize,
1423 i8, i16, i32, i64, i128, isize,
1424 f32, f64,
1425
1426 // Note: do not add uninhabited types (su
1427 // creating an instance of an uninhabited
1428 // uninhabited/empty types, consult The R
1429 // <https://doc.rust-lang.org/stable/nomi
1430 // also has information on undefined beha
1431 // <https://doc.rust-lang.org/stable/refe
1432 //
1433 // SAFETY: These are inhabited ZSTs; ther
1434 {<T: ?Sized>} PhantomData<T>, core::marke
1435
1436 // SAFETY: Type is allowed to take any va
1437 {<T>} MaybeUninit<T>,
1438 // SAFETY: Type is allowed to take any va
1439 {<T>} Opaque<T>,
1440
1441 // SAFETY: `T: Zeroable` and `UnsafeCell`
1442 {<T: ?Sized + Zeroable>} UnsafeCell<T>,
1443
1444 // SAFETY: All zeros is equivalent to `No
1445 Option<NonZeroU8>, Option<NonZeroU16>, Op
1446 Option<NonZeroU128>, Option<NonZeroUsize>
1447 Option<NonZeroI8>, Option<NonZeroI16>, Op
1448 Option<NonZeroI128>, Option<NonZeroIsize>
1449
1450 // SAFETY: All zeros is equivalent to `No
1451 //
1452 // In this case we are allowed to use `T:
1453 {<T: ?Sized>} Option<NonNull<T>>,
1454 {<T: ?Sized>} Option<Box<T>>,
1455
1456 // SAFETY: `null` pointer is valid.
1457 //
1458 // We cannot use `T: ?Sized`, since the V
1459 // null.
1460 //
1461 // When `Pointee` gets stabilized, we cou
1462 // `T: ?Sized where <T as Pointee>::Metad
1463 {<T>} *mut T, {<T>} *const T,
1464
1465 // SAFETY: `null` pointer is valid and th
1466 // zero.
1467 {<T>} *mut [T], {<T>} *const [T], *mut st
1468
1469 // SAFETY: `T` is `Zeroable`.
1470 {<const N: usize, T: Zeroable>} [T; N], {
1471 }
1472
1473 macro_rules! impl_tuple_zeroable {
1474 ($(,)?) => {};
1475 ($first:ident, $($t:ident),* $(,)?) => {
1476 // SAFETY: All elements are zeroable
1477 unsafe impl<$first: Zeroable, $($t: Z
1478 impl_tuple_zeroable!($($t),* ,);
1479 }
1480 }
1481
1482 impl_tuple_zeroable!(A, B, C, D, E, F, G, H,
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.