1 // SPDX-License-Identifier: Apache-2.0 OR MIT 1 // SPDX-License-Identifier: Apache-2.0 OR MIT
2 2
3 //! This module contains API-internal items fo 3 //! This module contains API-internal items for pin-init.
4 //! 4 //!
5 //! These items must not be used outside of 5 //! These items must not be used outside of
6 //! - `kernel/init.rs` 6 //! - `kernel/init.rs`
7 //! - `macros/pin_data.rs` 7 //! - `macros/pin_data.rs`
8 //! - `macros/pinned_drop.rs` 8 //! - `macros/pinned_drop.rs`
9 9
10 use super::*; 10 use super::*;
11 11
12 /// See the [nomicon] for what subtyping is. S 12 /// See the [nomicon] for what subtyping is. See also [this table].
13 /// 13 ///
14 /// [nomicon]: https://doc.rust-lang.org/nomic 14 /// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html
15 /// [this table]: https://doc.rust-lang.org/no 15 /// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns
16 pub(super) type Invariant<T> = PhantomData<fn( !! 16 type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>;
17 17
18 /// This is the module-internal type implement 18 /// This is the module-internal type implementing `PinInit` and `Init`. It is unsafe to create this
19 /// type, since the closure needs to fulfill t 19 /// type, since the closure needs to fulfill the same safety requirement as the
20 /// `__pinned_init`/`__init` functions. 20 /// `__pinned_init`/`__init` functions.
21 pub(crate) struct InitClosure<F, T: ?Sized, E> 21 pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>);
22 22
23 // SAFETY: While constructing the `InitClosure 23 // SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
24 // `__init` invariants. 24 // `__init` invariants.
25 unsafe impl<T: ?Sized, F, E> Init<T, E> for In 25 unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T, E>
26 where 26 where
27 F: FnOnce(*mut T) -> Result<(), E>, 27 F: FnOnce(*mut T) -> Result<(), E>,
28 { 28 {
29 #[inline] 29 #[inline]
30 unsafe fn __init(self, slot: *mut T) -> Re 30 unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
31 (self.0)(slot) 31 (self.0)(slot)
32 } 32 }
33 } 33 }
34 34
35 // SAFETY: While constructing the `InitClosure <<
36 // `__pinned_init` invariants. <<
37 unsafe impl<T: ?Sized, F, E> PinInit<T, E> for <<
38 where <<
39 F: FnOnce(*mut T) -> Result<(), E>, <<
40 { <<
41 #[inline] <<
42 unsafe fn __pinned_init(self, slot: *mut T <<
43 (self.0)(slot) <<
44 } <<
45 } <<
46 <<
47 /// This trait is only implemented via the `#[ 35 /// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate
48 /// the pin projections within the initializer 36 /// the pin projections within the initializers.
49 /// 37 ///
50 /// # Safety 38 /// # Safety
51 /// 39 ///
52 /// Only the `init` module is allowed to use t 40 /// Only the `init` module is allowed to use this trait.
53 pub unsafe trait HasPinData { 41 pub unsafe trait HasPinData {
54 type PinData: PinData; 42 type PinData: PinData;
55 43
56 unsafe fn __pin_data() -> Self::PinData; 44 unsafe fn __pin_data() -> Self::PinData;
57 } 45 }
58 46
59 /// Marker trait for pinning data of structs. 47 /// Marker trait for pinning data of structs.
60 /// 48 ///
61 /// # Safety 49 /// # Safety
62 /// 50 ///
63 /// Only the `init` module is allowed to use t 51 /// Only the `init` module is allowed to use this trait.
64 pub unsafe trait PinData: Copy { 52 pub unsafe trait PinData: Copy {
65 type Datee: ?Sized + HasPinData; 53 type Datee: ?Sized + HasPinData;
66 54
67 /// Type inference helper function. 55 /// Type inference helper function.
68 fn make_closure<F, O, E>(self, f: F) -> F 56 fn make_closure<F, O, E>(self, f: F) -> F
69 where 57 where
70 F: FnOnce(*mut Self::Datee) -> Result< 58 F: FnOnce(*mut Self::Datee) -> Result<O, E>,
71 { 59 {
72 f 60 f
73 } 61 }
74 } 62 }
75 63
76 /// This trait is automatically implemented fo 64 /// This trait is automatically implemented for every type. It aims to provide the same type
77 /// inference help as `HasPinData`. 65 /// inference help as `HasPinData`.
78 /// 66 ///
79 /// # Safety 67 /// # Safety
80 /// 68 ///
81 /// Only the `init` module is allowed to use t 69 /// Only the `init` module is allowed to use this trait.
82 pub unsafe trait HasInitData { 70 pub unsafe trait HasInitData {
83 type InitData: InitData; 71 type InitData: InitData;
84 72
85 unsafe fn __init_data() -> Self::InitData; 73 unsafe fn __init_data() -> Self::InitData;
86 } 74 }
87 75
88 /// Same function as `PinData`, but for arbitr 76 /// Same function as `PinData`, but for arbitrary data.
89 /// 77 ///
90 /// # Safety 78 /// # Safety
91 /// 79 ///
92 /// Only the `init` module is allowed to use t 80 /// Only the `init` module is allowed to use this trait.
93 pub unsafe trait InitData: Copy { 81 pub unsafe trait InitData: Copy {
94 type Datee: ?Sized + HasInitData; 82 type Datee: ?Sized + HasInitData;
95 83
96 /// Type inference helper function. 84 /// Type inference helper function.
97 fn make_closure<F, O, E>(self, f: F) -> F 85 fn make_closure<F, O, E>(self, f: F) -> F
98 where 86 where
99 F: FnOnce(*mut Self::Datee) -> Result< 87 F: FnOnce(*mut Self::Datee) -> Result<O, E>,
100 { 88 {
101 f 89 f
102 } 90 }
103 } 91 }
104 92
105 pub struct AllData<T: ?Sized>(PhantomData<fn(B 93 pub struct AllData<T: ?Sized>(PhantomData<fn(Box<T>) -> Box<T>>);
106 94
107 impl<T: ?Sized> Clone for AllData<T> { 95 impl<T: ?Sized> Clone for AllData<T> {
108 fn clone(&self) -> Self { 96 fn clone(&self) -> Self {
109 *self 97 *self
110 } 98 }
111 } 99 }
112 100
113 impl<T: ?Sized> Copy for AllData<T> {} 101 impl<T: ?Sized> Copy for AllData<T> {}
114 102
115 unsafe impl<T: ?Sized> InitData for AllData<T> 103 unsafe impl<T: ?Sized> InitData for AllData<T> {
116 type Datee = T; 104 type Datee = T;
117 } 105 }
118 106
119 unsafe impl<T: ?Sized> HasInitData for T { 107 unsafe impl<T: ?Sized> HasInitData for T {
120 type InitData = AllData<T>; 108 type InitData = AllData<T>;
121 109
122 unsafe fn __init_data() -> Self::InitData 110 unsafe fn __init_data() -> Self::InitData {
123 AllData(PhantomData) 111 AllData(PhantomData)
124 } 112 }
125 } 113 }
126 114
127 /// Stack initializer helper type. Use [`stack 115 /// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive.
128 /// 116 ///
129 /// # Invariants 117 /// # Invariants
130 /// 118 ///
131 /// If `self.is_init` is true, then `self.valu 119 /// If `self.is_init` is true, then `self.value` is initialized.
132 /// 120 ///
133 /// [`stack_pin_init`]: kernel::stack_pin_init 121 /// [`stack_pin_init`]: kernel::stack_pin_init
134 pub struct StackInit<T> { 122 pub struct StackInit<T> {
135 value: MaybeUninit<T>, 123 value: MaybeUninit<T>,
136 is_init: bool, 124 is_init: bool,
137 } 125 }
138 126
139 impl<T> Drop for StackInit<T> { 127 impl<T> Drop for StackInit<T> {
140 #[inline] 128 #[inline]
141 fn drop(&mut self) { 129 fn drop(&mut self) {
142 if self.is_init { 130 if self.is_init {
143 // SAFETY: As we are being dropped 131 // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is
144 // true, `self.value` is initializ 132 // true, `self.value` is initialized.
145 unsafe { self.value.assume_init_dr 133 unsafe { self.value.assume_init_drop() };
146 } 134 }
147 } 135 }
148 } 136 }
149 137
150 impl<T> StackInit<T> { 138 impl<T> StackInit<T> {
151 /// Creates a new [`StackInit<T>`] that is 139 /// Creates a new [`StackInit<T>`] that is uninitialized. Use [`stack_pin_init`] instead of this
152 /// primitive. 140 /// primitive.
153 /// 141 ///
154 /// [`stack_pin_init`]: kernel::stack_pin_ 142 /// [`stack_pin_init`]: kernel::stack_pin_init
155 #[inline] 143 #[inline]
156 pub fn uninit() -> Self { 144 pub fn uninit() -> Self {
157 Self { 145 Self {
158 value: MaybeUninit::uninit(), 146 value: MaybeUninit::uninit(),
159 is_init: false, 147 is_init: false,
160 } 148 }
161 } 149 }
162 150
163 /// Initializes the contents and returns t 151 /// Initializes the contents and returns the result.
164 #[inline] 152 #[inline]
165 pub fn init<E>(self: Pin<&mut Self>, init: 153 pub fn init<E>(self: Pin<&mut Self>, init: impl PinInit<T, E>) -> Result<Pin<&mut T>, E> {
166 // SAFETY: We never move out of `this` 154 // SAFETY: We never move out of `this`.
167 let this = unsafe { Pin::into_inner_un 155 let this = unsafe { Pin::into_inner_unchecked(self) };
168 // The value is currently initialized, 156 // The value is currently initialized, so it needs to be dropped before we can reuse
169 // the memory (this is a safety guaran 157 // the memory (this is a safety guarantee of `Pin`).
170 if this.is_init { 158 if this.is_init {
171 this.is_init = false; 159 this.is_init = false;
172 // SAFETY: `this.is_init` was true 160 // SAFETY: `this.is_init` was true and therefore `this.value` is initialized.
173 unsafe { this.value.assume_init_dr 161 unsafe { this.value.assume_init_drop() };
174 } 162 }
175 // SAFETY: The memory slot is valid an 163 // SAFETY: The memory slot is valid and this type ensures that it will stay pinned.
176 unsafe { init.__pinned_init(this.value 164 unsafe { init.__pinned_init(this.value.as_mut_ptr())? };
177 // INVARIANT: `this.value` is initiali 165 // INVARIANT: `this.value` is initialized above.
178 this.is_init = true; 166 this.is_init = true;
179 // SAFETY: The slot is now pinned, sin 167 // SAFETY: The slot is now pinned, since we will never give access to `&mut T`.
180 Ok(unsafe { Pin::new_unchecked(this.va 168 Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) })
181 } 169 }
182 } 170 }
183 171
184 /// When a value of this type is dropped, it d 172 /// When a value of this type is dropped, it drops a `T`.
185 /// 173 ///
186 /// Can be forgotten to prevent the drop. 174 /// Can be forgotten to prevent the drop.
187 pub struct DropGuard<T: ?Sized> { 175 pub struct DropGuard<T: ?Sized> {
188 ptr: *mut T, 176 ptr: *mut T,
>> 177 do_drop: Cell<bool>,
189 } 178 }
190 179
191 impl<T: ?Sized> DropGuard<T> { 180 impl<T: ?Sized> DropGuard<T> {
192 /// Creates a new [`DropGuard<T>`]. It wil 181 /// Creates a new [`DropGuard<T>`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped.
193 /// 182 ///
194 /// # Safety 183 /// # Safety
195 /// 184 ///
196 /// `ptr` must be a valid pointer. 185 /// `ptr` must be a valid pointer.
197 /// 186 ///
198 /// It is the callers responsibility that 187 /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`:
199 /// - has not been dropped, 188 /// - has not been dropped,
200 /// - is not accessible by any other means 189 /// - is not accessible by any other means,
201 /// - will not be dropped by any other mea 190 /// - will not be dropped by any other means.
202 #[inline] 191 #[inline]
203 pub unsafe fn new(ptr: *mut T) -> Self { 192 pub unsafe fn new(ptr: *mut T) -> Self {
204 Self { ptr } !! 193 Self {
>> 194 ptr,
>> 195 do_drop: Cell::new(true),
>> 196 }
>> 197 }
>> 198
>> 199 /// Prevents this guard from dropping the supplied pointer.
>> 200 ///
>> 201 /// # Safety
>> 202 ///
>> 203 /// This function is unsafe in order to prevent safe code from forgetting this guard. It should
>> 204 /// only be called by the macros in this module.
>> 205 #[inline]
>> 206 pub unsafe fn forget(&self) {
>> 207 self.do_drop.set(false);
205 } 208 }
206 } 209 }
207 210
208 impl<T: ?Sized> Drop for DropGuard<T> { 211 impl<T: ?Sized> Drop for DropGuard<T> {
209 #[inline] 212 #[inline]
210 fn drop(&mut self) { 213 fn drop(&mut self) {
211 // SAFETY: A `DropGuard` can only be c !! 214 if self.do_drop.get() {
212 // ensuring that this operation is saf !! 215 // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function
213 unsafe { ptr::drop_in_place(self.ptr) !! 216 // ensuring that this operation is safe.
>> 217 unsafe { ptr::drop_in_place(self.ptr) }
>> 218 }
214 } 219 }
215 } 220 }
216 221
217 /// Token used by `PinnedDrop` to prevent call 222 /// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely
218 /// created struct. This is needed, because th 223 /// created struct. This is needed, because the `drop` function is safe, but should not be called
219 /// manually. 224 /// manually.
220 pub struct OnlyCallFromDrop(()); 225 pub struct OnlyCallFromDrop(());
221 226
222 impl OnlyCallFromDrop { 227 impl OnlyCallFromDrop {
223 /// # Safety 228 /// # Safety
224 /// 229 ///
225 /// This function should only be called fr 230 /// This function should only be called from the [`Drop::drop`] function and only be used to
226 /// delegate the destruction to the pinned 231 /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type.
227 pub unsafe fn new() -> Self { 232 pub unsafe fn new() -> Self {
228 Self(()) 233 Self(())
229 } <<
230 } <<
231 <<
232 /// Initializer that always fails. <<
233 /// <<
234 /// Used by [`assert_pinned!`]. <<
235 /// <<
236 /// [`assert_pinned!`]: crate::assert_pinned <<
237 pub struct AlwaysFail<T: ?Sized> { <<
238 _t: PhantomData<T>, <<
239 } <<
240 <<
241 impl<T: ?Sized> AlwaysFail<T> { <<
242 /// Creates a new initializer that always <<
243 pub fn new() -> Self { <<
244 Self { _t: PhantomData } <<
245 } <<
246 } <<
247 <<
248 impl<T: ?Sized> Default for AlwaysFail<T> { <<
249 fn default() -> Self { <<
250 Self::new() <<
251 } <<
252 } <<
253 <<
254 // SAFETY: `__pinned_init` always fails, which <<
255 unsafe impl<T: ?Sized> PinInit<T, ()> for Alwa <<
256 unsafe fn __pinned_init(self, _slot: *mut <<
257 Err(()) <<
258 } 234 }
259 } 235 }
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.