1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 2
3 //! A reference-counted pointer. 3 //! A reference-counted pointer.
4 //! 4 //!
5 //! This module implements a way for users to 5 //! This module implements a way for users to create reference-counted objects and pointers to
6 //! them. Such a pointer automatically increme 6 //! them. Such a pointer automatically increments and decrements the count, and drops the
7 //! underlying object when it reaches zero. It 7 //! underlying object when it reaches zero. It is also safe to use concurrently from multiple
8 //! threads. 8 //! threads.
9 //! 9 //!
10 //! It is different from the standard library' 10 //! It is different from the standard library's [`Arc`] in a few ways:
11 //! 1. It is backed by the kernel's `refcount_ 11 //! 1. It is backed by the kernel's `refcount_t` type.
12 //! 2. It does not support weak references, wh 12 //! 2. It does not support weak references, which allows it to be half the size.
13 //! 3. It saturates the reference count instea 13 //! 3. It saturates the reference count instead of aborting when it goes over a threshold.
14 //! 4. It does not provide a `get_mut` method, 14 //! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
15 //! 5. The object in [`Arc`] is pinned implici <<
16 //! 15 //!
17 //! [`Arc`]: https://doc.rust-lang.org/std/syn 16 //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
18 17
19 use crate::{ 18 use crate::{
20 alloc::{box_ext::BoxExt, AllocError, Flags 19 alloc::{box_ext::BoxExt, AllocError, Flags},
21 bindings, !! 20 error::{self, Error},
22 init::{self, InPlaceInit, Init, PinInit}, 21 init::{self, InPlaceInit, Init, PinInit},
23 try_init, 22 try_init,
24 types::{ForeignOwnable, Opaque}, 23 types::{ForeignOwnable, Opaque},
25 }; 24 };
26 use alloc::boxed::Box; 25 use alloc::boxed::Box;
27 use core::{ 26 use core::{
28 alloc::Layout, 27 alloc::Layout,
29 fmt, 28 fmt,
30 marker::{PhantomData, Unsize}, 29 marker::{PhantomData, Unsize},
31 mem::{ManuallyDrop, MaybeUninit}, 30 mem::{ManuallyDrop, MaybeUninit},
32 ops::{Deref, DerefMut}, 31 ops::{Deref, DerefMut},
33 pin::Pin, 32 pin::Pin,
34 ptr::NonNull, 33 ptr::NonNull,
35 }; 34 };
36 use macros::pin_data; 35 use macros::pin_data;
37 36
38 mod std_vendor; 37 mod std_vendor;
39 38
40 /// A reference-counted pointer to an instance 39 /// A reference-counted pointer to an instance of `T`.
41 /// 40 ///
42 /// The reference count is incremented when ne 41 /// The reference count is incremented when new instances of [`Arc`] are created, and decremented
43 /// when they are dropped. When the count reac 42 /// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
44 /// 43 ///
45 /// # Invariants 44 /// # Invariants
46 /// 45 ///
47 /// The reference count on an instance of [`Ar 46 /// The reference count on an instance of [`Arc`] is always non-zero.
48 /// The object pointed to by [`Arc`] is always 47 /// The object pointed to by [`Arc`] is always pinned.
49 /// 48 ///
50 /// # Examples 49 /// # Examples
51 /// 50 ///
52 /// ``` 51 /// ```
53 /// use kernel::sync::Arc; 52 /// use kernel::sync::Arc;
54 /// 53 ///
55 /// struct Example { 54 /// struct Example {
56 /// a: u32, 55 /// a: u32,
57 /// b: u32, 56 /// b: u32,
58 /// } 57 /// }
59 /// 58 ///
60 /// // Create a refcounted instance of `Exampl 59 /// // Create a refcounted instance of `Example`.
61 /// let obj = Arc::new(Example { a: 10, b: 20 60 /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
62 /// 61 ///
63 /// // Get a new pointer to `obj` and incremen 62 /// // Get a new pointer to `obj` and increment the refcount.
64 /// let cloned = obj.clone(); 63 /// let cloned = obj.clone();
65 /// 64 ///
66 /// // Assert that both `obj` and `cloned` poi 65 /// // Assert that both `obj` and `cloned` point to the same underlying object.
67 /// assert!(core::ptr::eq(&*obj, &*cloned)); 66 /// assert!(core::ptr::eq(&*obj, &*cloned));
68 /// 67 ///
69 /// // Destroy `obj` and decrement its refcoun 68 /// // Destroy `obj` and decrement its refcount.
70 /// drop(obj); 69 /// drop(obj);
71 /// 70 ///
72 /// // Check that the values are still accessi 71 /// // Check that the values are still accessible through `cloned`.
73 /// assert_eq!(cloned.a, 10); 72 /// assert_eq!(cloned.a, 10);
74 /// assert_eq!(cloned.b, 20); 73 /// assert_eq!(cloned.b, 20);
75 /// 74 ///
76 /// // The refcount drops to zero when `cloned 75 /// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
77 /// # Ok::<(), Error>(()) 76 /// # Ok::<(), Error>(())
78 /// ``` 77 /// ```
79 /// 78 ///
80 /// Using `Arc<T>` as the type of `self`: 79 /// Using `Arc<T>` as the type of `self`:
81 /// 80 ///
82 /// ``` 81 /// ```
83 /// use kernel::sync::Arc; 82 /// use kernel::sync::Arc;
84 /// 83 ///
85 /// struct Example { 84 /// struct Example {
86 /// a: u32, 85 /// a: u32,
87 /// b: u32, 86 /// b: u32,
88 /// } 87 /// }
89 /// 88 ///
90 /// impl Example { 89 /// impl Example {
91 /// fn take_over(self: Arc<Self>) { 90 /// fn take_over(self: Arc<Self>) {
92 /// // ... 91 /// // ...
93 /// } 92 /// }
94 /// 93 ///
95 /// fn use_reference(self: &Arc<Self>) { 94 /// fn use_reference(self: &Arc<Self>) {
96 /// // ... 95 /// // ...
97 /// } 96 /// }
98 /// } 97 /// }
99 /// 98 ///
100 /// let obj = Arc::new(Example { a: 10, b: 20 99 /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
101 /// obj.use_reference(); 100 /// obj.use_reference();
102 /// obj.take_over(); 101 /// obj.take_over();
103 /// # Ok::<(), Error>(()) 102 /// # Ok::<(), Error>(())
104 /// ``` 103 /// ```
105 /// 104 ///
106 /// Coercion from `Arc<Example>` to `Arc<dyn M 105 /// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
107 /// 106 ///
108 /// ``` 107 /// ```
109 /// use kernel::sync::{Arc, ArcBorrow}; 108 /// use kernel::sync::{Arc, ArcBorrow};
110 /// 109 ///
111 /// trait MyTrait { 110 /// trait MyTrait {
112 /// // Trait has a function whose `self` t 111 /// // Trait has a function whose `self` type is `Arc<Self>`.
113 /// fn example1(self: Arc<Self>) {} 112 /// fn example1(self: Arc<Self>) {}
114 /// 113 ///
115 /// // Trait has a function whose `self` t 114 /// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
116 /// fn example2(self: ArcBorrow<'_, Self>) 115 /// fn example2(self: ArcBorrow<'_, Self>) {}
117 /// } 116 /// }
118 /// 117 ///
119 /// struct Example; 118 /// struct Example;
120 /// impl MyTrait for Example {} 119 /// impl MyTrait for Example {}
121 /// 120 ///
122 /// // `obj` has type `Arc<Example>`. 121 /// // `obj` has type `Arc<Example>`.
123 /// let obj: Arc<Example> = Arc::new(Example, 122 /// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?;
124 /// 123 ///
125 /// // `coerced` has type `Arc<dyn MyTrait>`. 124 /// // `coerced` has type `Arc<dyn MyTrait>`.
126 /// let coerced: Arc<dyn MyTrait> = obj; 125 /// let coerced: Arc<dyn MyTrait> = obj;
127 /// # Ok::<(), Error>(()) 126 /// # Ok::<(), Error>(())
128 /// ``` 127 /// ```
129 pub struct Arc<T: ?Sized> { 128 pub struct Arc<T: ?Sized> {
130 ptr: NonNull<ArcInner<T>>, 129 ptr: NonNull<ArcInner<T>>,
131 _p: PhantomData<ArcInner<T>>, 130 _p: PhantomData<ArcInner<T>>,
132 } 131 }
133 132
134 #[pin_data] 133 #[pin_data]
135 #[repr(C)] 134 #[repr(C)]
136 struct ArcInner<T: ?Sized> { 135 struct ArcInner<T: ?Sized> {
137 refcount: Opaque<bindings::refcount_t>, 136 refcount: Opaque<bindings::refcount_t>,
138 data: T, 137 data: T,
139 } 138 }
140 139
141 impl<T: ?Sized> ArcInner<T> { 140 impl<T: ?Sized> ArcInner<T> {
142 /// Converts a pointer to the contents of 141 /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`].
143 /// 142 ///
144 /// # Safety 143 /// # Safety
145 /// 144 ///
146 /// `ptr` must have been returned by a pre 145 /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must
147 /// not yet have been destroyed. 146 /// not yet have been destroyed.
148 unsafe fn container_of(ptr: *const T) -> N 147 unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> {
149 let refcount_layout = Layout::new::<bi 148 let refcount_layout = Layout::new::<bindings::refcount_t>();
150 // SAFETY: The caller guarantees that 149 // SAFETY: The caller guarantees that the pointer is valid.
151 let val_layout = Layout::for_value(uns 150 let val_layout = Layout::for_value(unsafe { &*ptr });
152 // SAFETY: We're computing the layout 151 // SAFETY: We're computing the layout of a real struct that existed when compiling this
153 // binary, so its layout is not so lar 152 // binary, so its layout is not so large that it can trigger arithmetic overflow.
154 let val_offset = unsafe { refcount_lay 153 let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 };
155 154
156 // Pointer casts leave the metadata un 155 // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and
157 // `ArcInner<T>` is the same since `Ar 156 // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field.
158 // 157 //
159 // This is documented at: 158 // This is documented at:
160 // <https://doc.rust-lang.org/std/ptr/ 159 // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>.
161 let ptr = ptr as *const ArcInner<T>; 160 let ptr = ptr as *const ArcInner<T>;
162 161
163 // SAFETY: The pointer is in-bounds of 162 // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the
164 // pointer, since it originates from a 163 // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is
165 // still valid. 164 // still valid.
166 let ptr = unsafe { ptr.byte_sub(val_of 165 let ptr = unsafe { ptr.byte_sub(val_offset) };
167 166
168 // SAFETY: The pointer can't be null s 167 // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null
169 // address. 168 // address.
170 unsafe { NonNull::new_unchecked(ptr.ca 169 unsafe { NonNull::new_unchecked(ptr.cast_mut()) }
171 } 170 }
172 } 171 }
173 172
174 // This is to allow [`Arc`] (and variants) to 173 // This is to allow [`Arc`] (and variants) to be used as the type of `self`.
175 impl<T: ?Sized> core::ops::Receiver for Arc<T> 174 impl<T: ?Sized> core::ops::Receiver for Arc<T> {}
176 175
177 // This is to allow coercion from `Arc<T>` to 176 // This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
178 // dynamically-sized type (DST) `U`. 177 // dynamically-sized type (DST) `U`.
179 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::o 178 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
180 179
181 // This is to allow `Arc<U>` to be dispatched 180 // This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
182 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::o 181 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
183 182
184 // SAFETY: It is safe to send `Arc<T>` to anot 183 // SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
185 // it effectively means sharing `&T` (which is 184 // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
186 // `T` to be `Send` because any thread that ha 185 // `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a
187 // mutable reference when the reference count 186 // mutable reference when the reference count reaches zero and `T` is dropped.
188 unsafe impl<T: ?Sized + Sync + Send> Send for 187 unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
189 188
190 // SAFETY: It is safe to send `&Arc<T>` to ano 189 // SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync`
191 // because it effectively means sharing `&T` ( 190 // because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally,
192 // it needs `T` to be `Send` because any threa 191 // it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an
193 // `Arc<T>` on that thread, so the thread may 192 // `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when
194 // the reference count reaches zero and `T` is 193 // the reference count reaches zero and `T` is dropped.
195 unsafe impl<T: ?Sized + Sync + Send> Sync for 194 unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
196 195
197 impl<T> Arc<T> { 196 impl<T> Arc<T> {
198 /// Constructs a new reference counted ins 197 /// Constructs a new reference counted instance of `T`.
199 pub fn new(contents: T, flags: Flags) -> R 198 pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> {
200 // INVARIANT: The refcount is initiali 199 // INVARIANT: The refcount is initialised to a non-zero value.
201 let value = ArcInner { 200 let value = ArcInner {
202 // SAFETY: There are no safety req 201 // SAFETY: There are no safety requirements for this FFI call.
203 refcount: Opaque::new(unsafe { bin 202 refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
204 data: contents, 203 data: contents,
205 }; 204 };
206 205
207 let inner = <Box<_> as BoxExt<_>>::new 206 let inner = <Box<_> as BoxExt<_>>::new(value, flags)?;
208 207
209 // SAFETY: We just created `inner` wit 208 // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
210 // `Arc` object. 209 // `Arc` object.
211 Ok(unsafe { Self::from_inner(Box::leak 210 Ok(unsafe { Self::from_inner(Box::leak(inner).into()) })
>> 211 }
>> 212
>> 213 /// Use the given initializer to in-place initialize a `T`.
>> 214 ///
>> 215 /// If `T: !Unpin` it will not be able to move afterwards.
>> 216 #[inline]
>> 217 pub fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self>
>> 218 where
>> 219 Error: From<E>,
>> 220 {
>> 221 UniqueArc::pin_init(init, flags).map(|u| u.into())
>> 222 }
>> 223
>> 224 /// Use the given initializer to in-place initialize a `T`.
>> 225 ///
>> 226 /// This is equivalent to [`Arc<T>::pin_init`], since an [`Arc`] is always pinned.
>> 227 #[inline]
>> 228 pub fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
>> 229 where
>> 230 Error: From<E>,
>> 231 {
>> 232 UniqueArc::init(init, flags).map(|u| u.into())
212 } 233 }
213 } 234 }
214 235
215 impl<T: ?Sized> Arc<T> { 236 impl<T: ?Sized> Arc<T> {
216 /// Constructs a new [`Arc`] from an exist 237 /// Constructs a new [`Arc`] from an existing [`ArcInner`].
217 /// 238 ///
218 /// # Safety 239 /// # Safety
219 /// 240 ///
220 /// The caller must ensure that `inner` po 241 /// The caller must ensure that `inner` points to a valid location and has a non-zero reference
221 /// count, one of which will be owned by t 242 /// count, one of which will be owned by the new [`Arc`] instance.
222 unsafe fn from_inner(inner: NonNull<ArcInn 243 unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
223 // INVARIANT: By the safety requiremen 244 // INVARIANT: By the safety requirements, the invariants hold.
224 Arc { 245 Arc {
225 ptr: inner, 246 ptr: inner,
226 _p: PhantomData, 247 _p: PhantomData,
227 } 248 }
228 } 249 }
229 250
230 /// Convert the [`Arc`] into a raw pointer 251 /// Convert the [`Arc`] into a raw pointer.
231 /// 252 ///
232 /// The raw pointer has ownership of the r 253 /// The raw pointer has ownership of the refcount that this Arc object owned.
233 pub fn into_raw(self) -> *const T { 254 pub fn into_raw(self) -> *const T {
234 let ptr = self.ptr.as_ptr(); 255 let ptr = self.ptr.as_ptr();
235 core::mem::forget(self); 256 core::mem::forget(self);
236 // SAFETY: The pointer is valid. 257 // SAFETY: The pointer is valid.
237 unsafe { core::ptr::addr_of!((*ptr).da 258 unsafe { core::ptr::addr_of!((*ptr).data) }
238 } 259 }
239 260
240 /// Recreates an [`Arc`] instance previous 261 /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`].
241 /// 262 ///
242 /// # Safety 263 /// # Safety
243 /// 264 ///
244 /// `ptr` must have been returned by a pre 265 /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it
245 /// must not be called more than once for 266 /// must not be called more than once for each previous call to [`Arc::into_raw`].
246 pub unsafe fn from_raw(ptr: *const T) -> S 267 pub unsafe fn from_raw(ptr: *const T) -> Self {
247 // SAFETY: The caller promises that th 268 // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
248 // `Arc` that is still valid. 269 // `Arc` that is still valid.
249 let ptr = unsafe { ArcInner::container 270 let ptr = unsafe { ArcInner::container_of(ptr) };
250 271
251 // SAFETY: By the safety requirements 272 // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the
252 // reference count held then will be o 273 // reference count held then will be owned by the new `Arc` object.
253 unsafe { Self::from_inner(ptr) } 274 unsafe { Self::from_inner(ptr) }
254 } 275 }
255 276
256 /// Returns an [`ArcBorrow`] from the give 277 /// Returns an [`ArcBorrow`] from the given [`Arc`].
257 /// 278 ///
258 /// This is useful when the argument of a 279 /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
259 /// receiver), but we have an [`Arc`] inst 280 /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
260 #[inline] 281 #[inline]
261 pub fn as_arc_borrow(&self) -> ArcBorrow<' 282 pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
262 // SAFETY: The constraint that the lif 283 // SAFETY: The constraint that the lifetime of the shared reference must outlive that of
263 // the returned `ArcBorrow` ensures th 284 // the returned `ArcBorrow` ensures that the object remains alive and that no mutable
264 // reference can be created. 285 // reference can be created.
265 unsafe { ArcBorrow::new(self.ptr) } 286 unsafe { ArcBorrow::new(self.ptr) }
266 } 287 }
267 288
268 /// Compare whether two [`Arc`] pointers r 289 /// Compare whether two [`Arc`] pointers reference the same underlying object.
269 pub fn ptr_eq(this: &Self, other: &Self) - 290 pub fn ptr_eq(this: &Self, other: &Self) -> bool {
270 core::ptr::eq(this.ptr.as_ptr(), other 291 core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())
271 } 292 }
272 293
273 /// Converts this [`Arc`] into a [`UniqueA 294 /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique.
274 /// 295 ///
275 /// When this destroys the `Arc`, it does 296 /// When this destroys the `Arc`, it does so while properly avoiding races. This means that
276 /// this method will never call the destru 297 /// this method will never call the destructor of the value.
277 /// 298 ///
278 /// # Examples 299 /// # Examples
279 /// 300 ///
280 /// ``` 301 /// ```
281 /// use kernel::sync::{Arc, UniqueArc}; 302 /// use kernel::sync::{Arc, UniqueArc};
282 /// 303 ///
283 /// let arc = Arc::new(42, GFP_KERNEL)?; 304 /// let arc = Arc::new(42, GFP_KERNEL)?;
284 /// let unique_arc = arc.into_unique_or_dr 305 /// let unique_arc = arc.into_unique_or_drop();
285 /// 306 ///
286 /// // The above conversion should succeed 307 /// // The above conversion should succeed since refcount of `arc` is 1.
287 /// assert!(unique_arc.is_some()); 308 /// assert!(unique_arc.is_some());
288 /// 309 ///
289 /// assert_eq!(*(unique_arc.unwrap()), 42) 310 /// assert_eq!(*(unique_arc.unwrap()), 42);
290 /// 311 ///
291 /// # Ok::<(), Error>(()) 312 /// # Ok::<(), Error>(())
292 /// ``` 313 /// ```
293 /// 314 ///
294 /// ``` 315 /// ```
295 /// use kernel::sync::{Arc, UniqueArc}; 316 /// use kernel::sync::{Arc, UniqueArc};
296 /// 317 ///
297 /// let arc = Arc::new(42, GFP_KERNEL)?; 318 /// let arc = Arc::new(42, GFP_KERNEL)?;
298 /// let another = arc.clone(); 319 /// let another = arc.clone();
299 /// 320 ///
300 /// let unique_arc = arc.into_unique_or_dr 321 /// let unique_arc = arc.into_unique_or_drop();
301 /// 322 ///
302 /// // The above conversion should fail si 323 /// // The above conversion should fail since refcount of `arc` is >1.
303 /// assert!(unique_arc.is_none()); 324 /// assert!(unique_arc.is_none());
304 /// 325 ///
305 /// # Ok::<(), Error>(()) 326 /// # Ok::<(), Error>(())
306 /// ``` 327 /// ```
307 pub fn into_unique_or_drop(self) -> Option 328 pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> {
308 // We will manually manage the refcoun 329 // We will manually manage the refcount in this method, so we disable the destructor.
309 let me = ManuallyDrop::new(self); 330 let me = ManuallyDrop::new(self);
310 // SAFETY: We own a refcount, so the p 331 // SAFETY: We own a refcount, so the pointer is still valid.
311 let refcount = unsafe { me.ptr.as_ref( 332 let refcount = unsafe { me.ptr.as_ref() }.refcount.get();
312 333
313 // If the refcount reaches a non-zero 334 // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will
314 // return without further touching the 335 // return without further touching the `Arc`. If the refcount reaches zero, then there are
315 // no other arcs, and we can create a 336 // no other arcs, and we can create a `UniqueArc`.
316 // 337 //
317 // SAFETY: We own a refcount, so the p 338 // SAFETY: We own a refcount, so the pointer is not dangling.
318 let is_zero = unsafe { bindings::refco 339 let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
319 if is_zero { 340 if is_zero {
320 // SAFETY: We have exclusive acces 341 // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized
321 // accesses to the refcount. 342 // accesses to the refcount.
322 unsafe { core::ptr::write(refcount 343 unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) };
323 344
324 // INVARIANT: We own the only refc 345 // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We
325 // must pin the `UniqueArc` becaus 346 // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin
326 // their values. 347 // their values.
327 Some(Pin::from(UniqueArc { 348 Some(Pin::from(UniqueArc {
328 inner: ManuallyDrop::into_inne 349 inner: ManuallyDrop::into_inner(me),
329 })) 350 }))
330 } else { 351 } else {
331 None 352 None
332 } 353 }
333 } 354 }
334 } 355 }
335 356
336 impl<T: 'static> ForeignOwnable for Arc<T> { 357 impl<T: 'static> ForeignOwnable for Arc<T> {
337 type Borrowed<'a> = ArcBorrow<'a, T>; 358 type Borrowed<'a> = ArcBorrow<'a, T>;
338 359
339 fn into_foreign(self) -> *const core::ffi: 360 fn into_foreign(self) -> *const core::ffi::c_void {
340 ManuallyDrop::new(self).ptr.as_ptr() a 361 ManuallyDrop::new(self).ptr.as_ptr() as _
341 } 362 }
342 363
343 unsafe fn borrow<'a>(ptr: *const core::ffi 364 unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> ArcBorrow<'a, T> {
344 // SAFETY: By the safety requirement o 365 // SAFETY: By the safety requirement of this function, we know that `ptr` came from
345 // a previous call to `Arc::into_forei 366 // a previous call to `Arc::into_foreign`.
346 let inner = NonNull::new(ptr as *mut A 367 let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap();
347 368
348 // SAFETY: The safety requirements of 369 // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
349 // for the lifetime of the returned va 370 // for the lifetime of the returned value.
350 unsafe { ArcBorrow::new(inner) } 371 unsafe { ArcBorrow::new(inner) }
351 } 372 }
352 373
353 unsafe fn from_foreign(ptr: *const core::f 374 unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self {
354 // SAFETY: By the safety requirement o 375 // SAFETY: By the safety requirement of this function, we know that `ptr` came from
355 // a previous call to `Arc::into_forei 376 // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
356 // holds a reference count increment t 377 // holds a reference count increment that is transferrable to us.
357 unsafe { Self::from_inner(NonNull::new 378 unsafe { Self::from_inner(NonNull::new(ptr as _).unwrap()) }
358 } 379 }
359 } 380 }
360 381
361 impl<T: ?Sized> Deref for Arc<T> { 382 impl<T: ?Sized> Deref for Arc<T> {
362 type Target = T; 383 type Target = T;
363 384
364 fn deref(&self) -> &Self::Target { 385 fn deref(&self) -> &Self::Target {
365 // SAFETY: By the type invariant, ther 386 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
366 // safe to dereference it. 387 // safe to dereference it.
367 unsafe { &self.ptr.as_ref().data } 388 unsafe { &self.ptr.as_ref().data }
368 } 389 }
369 } 390 }
370 391
371 impl<T: ?Sized> AsRef<T> for Arc<T> { 392 impl<T: ?Sized> AsRef<T> for Arc<T> {
372 fn as_ref(&self) -> &T { 393 fn as_ref(&self) -> &T {
373 self.deref() 394 self.deref()
374 } 395 }
375 } 396 }
376 397
377 impl<T: ?Sized> Clone for Arc<T> { 398 impl<T: ?Sized> Clone for Arc<T> {
378 fn clone(&self) -> Self { 399 fn clone(&self) -> Self {
379 // INVARIANT: C `refcount_inc` saturat 400 // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero.
380 // SAFETY: By the type invariant, ther 401 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
381 // safe to increment the refcount. 402 // safe to increment the refcount.
382 unsafe { bindings::refcount_inc(self.p 403 unsafe { bindings::refcount_inc(self.ptr.as_ref().refcount.get()) };
383 404
384 // SAFETY: We just incremented the ref 405 // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
385 unsafe { Self::from_inner(self.ptr) } 406 unsafe { Self::from_inner(self.ptr) }
386 } 407 }
387 } 408 }
388 409
389 impl<T: ?Sized> Drop for Arc<T> { 410 impl<T: ?Sized> Drop for Arc<T> {
390 fn drop(&mut self) { 411 fn drop(&mut self) {
391 // SAFETY: By the type invariant, ther 412 // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot
392 // touch `refcount` after it's decreme 413 // touch `refcount` after it's decremented to a non-zero value because another thread/CPU
393 // may concurrently decrement it to ze 414 // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to
394 // freed/invalid memory as long as it 415 // freed/invalid memory as long as it is never dereferenced.
395 let refcount = unsafe { self.ptr.as_re 416 let refcount = unsafe { self.ptr.as_ref() }.refcount.get();
396 417
397 // INVARIANT: If the refcount reaches 418 // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
398 // this instance is being dropped, so 419 // this instance is being dropped, so the broken invariant is not observable.
399 // SAFETY: Also by the type invariant, 420 // SAFETY: Also by the type invariant, we are allowed to decrement the refcount.
400 let is_zero = unsafe { bindings::refco 421 let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
401 if is_zero { 422 if is_zero {
402 // The count reached zero, we must 423 // The count reached zero, we must free the memory.
403 // 424 //
404 // SAFETY: The pointer was initial 425 // SAFETY: The pointer was initialised from the result of `Box::leak`.
405 unsafe { drop(Box::from_raw(self.p 426 unsafe { drop(Box::from_raw(self.ptr.as_ptr())) };
406 } 427 }
407 } 428 }
408 } 429 }
409 430
410 impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> 431 impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
411 fn from(item: UniqueArc<T>) -> Self { 432 fn from(item: UniqueArc<T>) -> Self {
412 item.inner 433 item.inner
413 } 434 }
414 } 435 }
415 436
416 impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Ar 437 impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
417 fn from(item: Pin<UniqueArc<T>>) -> Self { 438 fn from(item: Pin<UniqueArc<T>>) -> Self {
418 // SAFETY: The type invariants of `Arc 439 // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
419 unsafe { Pin::into_inner_unchecked(ite 440 unsafe { Pin::into_inner_unchecked(item).inner }
420 } 441 }
421 } 442 }
422 443
423 /// A borrowed reference to an [`Arc`] instanc 444 /// A borrowed reference to an [`Arc`] instance.
424 /// 445 ///
425 /// For cases when one doesn't ever need to in 446 /// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
426 /// to use just `&T`, which we can trivially g 447 /// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance.
427 /// 448 ///
428 /// However, when one may need to increment th 449 /// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
429 /// over `&Arc<T>` because the latter results 450 /// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
430 /// to a pointer ([`Arc<T>`]) to the object (` 451 /// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double
431 /// indirection while still allowing one to in 452 /// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if
432 /// needed. 453 /// needed.
433 /// 454 ///
434 /// # Invariants 455 /// # Invariants
435 /// 456 ///
436 /// There are no mutable references to the und 457 /// There are no mutable references to the underlying [`Arc`], and it remains valid for the
437 /// lifetime of the [`ArcBorrow`] instance. 458 /// lifetime of the [`ArcBorrow`] instance.
438 /// 459 ///
439 /// # Example 460 /// # Example
440 /// 461 ///
441 /// ``` 462 /// ```
442 /// use kernel::sync::{Arc, ArcBorrow}; 463 /// use kernel::sync::{Arc, ArcBorrow};
443 /// 464 ///
444 /// struct Example; 465 /// struct Example;
445 /// 466 ///
446 /// fn do_something(e: ArcBorrow<'_, Example>) 467 /// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
447 /// e.into() 468 /// e.into()
448 /// } 469 /// }
449 /// 470 ///
450 /// let obj = Arc::new(Example, GFP_KERNEL)?; 471 /// let obj = Arc::new(Example, GFP_KERNEL)?;
451 /// let cloned = do_something(obj.as_arc_borro 472 /// let cloned = do_something(obj.as_arc_borrow());
452 /// 473 ///
453 /// // Assert that both `obj` and `cloned` poi 474 /// // Assert that both `obj` and `cloned` point to the same underlying object.
454 /// assert!(core::ptr::eq(&*obj, &*cloned)); 475 /// assert!(core::ptr::eq(&*obj, &*cloned));
455 /// # Ok::<(), Error>(()) 476 /// # Ok::<(), Error>(())
456 /// ``` 477 /// ```
457 /// 478 ///
458 /// Using `ArcBorrow<T>` as the type of `self` 479 /// Using `ArcBorrow<T>` as the type of `self`:
459 /// 480 ///
460 /// ``` 481 /// ```
461 /// use kernel::sync::{Arc, ArcBorrow}; 482 /// use kernel::sync::{Arc, ArcBorrow};
462 /// 483 ///
463 /// struct Example { 484 /// struct Example {
464 /// a: u32, 485 /// a: u32,
465 /// b: u32, 486 /// b: u32,
466 /// } 487 /// }
467 /// 488 ///
468 /// impl Example { 489 /// impl Example {
469 /// fn use_reference(self: ArcBorrow<'_, S 490 /// fn use_reference(self: ArcBorrow<'_, Self>) {
470 /// // ... 491 /// // ...
471 /// } 492 /// }
472 /// } 493 /// }
473 /// 494 ///
474 /// let obj = Arc::new(Example { a: 10, b: 20 495 /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
475 /// obj.as_arc_borrow().use_reference(); 496 /// obj.as_arc_borrow().use_reference();
476 /// # Ok::<(), Error>(()) 497 /// # Ok::<(), Error>(())
477 /// ``` 498 /// ```
478 pub struct ArcBorrow<'a, T: ?Sized + 'a> { 499 pub struct ArcBorrow<'a, T: ?Sized + 'a> {
479 inner: NonNull<ArcInner<T>>, 500 inner: NonNull<ArcInner<T>>,
480 _p: PhantomData<&'a ()>, 501 _p: PhantomData<&'a ()>,
481 } 502 }
482 503
483 // This is to allow [`ArcBorrow`] (and variant 504 // This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`.
484 impl<T: ?Sized> core::ops::Receiver for ArcBor 505 impl<T: ?Sized> core::ops::Receiver for ArcBorrow<'_, T> {}
485 506
486 // This is to allow `ArcBorrow<U>` to be dispa 507 // This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
487 // `ArcBorrow<U>`. 508 // `ArcBorrow<U>`.
488 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::o 509 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
489 for ArcBorrow<'_, T> 510 for ArcBorrow<'_, T>
490 { 511 {
491 } 512 }
492 513
493 impl<T: ?Sized> Clone for ArcBorrow<'_, T> { 514 impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
494 fn clone(&self) -> Self { 515 fn clone(&self) -> Self {
495 *self 516 *self
496 } 517 }
497 } 518 }
498 519
499 impl<T: ?Sized> Copy for ArcBorrow<'_, T> {} 520 impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
500 521
501 impl<T: ?Sized> ArcBorrow<'_, T> { 522 impl<T: ?Sized> ArcBorrow<'_, T> {
502 /// Creates a new [`ArcBorrow`] instance. 523 /// Creates a new [`ArcBorrow`] instance.
503 /// 524 ///
504 /// # Safety 525 /// # Safety
505 /// 526 ///
506 /// Callers must ensure the following for 527 /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
507 /// 1. That `inner` remains valid; 528 /// 1. That `inner` remains valid;
508 /// 2. That no mutable references to `inne 529 /// 2. That no mutable references to `inner` are created.
509 unsafe fn new(inner: NonNull<ArcInner<T>>) 530 unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
510 // INVARIANT: The safety requirements 531 // INVARIANT: The safety requirements guarantee the invariants.
511 Self { 532 Self {
512 inner, 533 inner,
513 _p: PhantomData, 534 _p: PhantomData,
514 } 535 }
515 } 536 }
516 537
517 /// Creates an [`ArcBorrow`] to an [`Arc`] 538 /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with
518 /// [`Arc::into_raw`]. 539 /// [`Arc::into_raw`].
519 /// 540 ///
520 /// # Safety 541 /// # Safety
521 /// 542 ///
522 /// * The provided pointer must originate 543 /// * The provided pointer must originate from a call to [`Arc::into_raw`].
523 /// * For the duration of the lifetime ann 544 /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must
524 /// not hit zero. 545 /// not hit zero.
525 /// * For the duration of the lifetime ann 546 /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a
526 /// [`UniqueArc`] reference to this valu 547 /// [`UniqueArc`] reference to this value.
527 pub unsafe fn from_raw(ptr: *const T) -> S 548 pub unsafe fn from_raw(ptr: *const T) -> Self {
528 // SAFETY: The caller promises that th 549 // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an
529 // `Arc` that is still valid. 550 // `Arc` that is still valid.
530 let ptr = unsafe { ArcInner::container 551 let ptr = unsafe { ArcInner::container_of(ptr) };
531 552
532 // SAFETY: The caller promises that th 553 // SAFETY: The caller promises that the value remains valid since the reference count must
533 // not hit zero, and no mutable refere 554 // not hit zero, and no mutable reference will be created since that would involve a
534 // `UniqueArc`. 555 // `UniqueArc`.
535 unsafe { Self::new(ptr) } 556 unsafe { Self::new(ptr) }
536 } 557 }
537 } 558 }
538 559
539 impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc 560 impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
540 fn from(b: ArcBorrow<'_, T>) -> Self { 561 fn from(b: ArcBorrow<'_, T>) -> Self {
541 // SAFETY: The existence of `b` guaran 562 // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
542 // guarantees that `drop` isn't called 563 // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
543 // increment. 564 // increment.
544 ManuallyDrop::new(unsafe { Arc::from_i 565 ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
545 .deref() 566 .deref()
546 .clone() 567 .clone()
547 } 568 }
548 } 569 }
549 570
550 impl<T: ?Sized> Deref for ArcBorrow<'_, T> { 571 impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
551 type Target = T; 572 type Target = T;
552 573
553 fn deref(&self) -> &Self::Target { 574 fn deref(&self) -> &Self::Target {
554 // SAFETY: By the type invariant, the 575 // SAFETY: By the type invariant, the underlying object is still alive with no mutable
555 // references to it, so it is safe to 576 // references to it, so it is safe to create a shared reference.
556 unsafe { &self.inner.as_ref().data } 577 unsafe { &self.inner.as_ref().data }
557 } 578 }
558 } 579 }
559 580
560 /// A refcounted object that is known to have 581 /// A refcounted object that is known to have a refcount of 1.
561 /// 582 ///
562 /// It is mutable and can be converted to an [ 583 /// It is mutable and can be converted to an [`Arc`] so that it can be shared.
563 /// 584 ///
564 /// # Invariants 585 /// # Invariants
565 /// 586 ///
566 /// `inner` always has a reference count of 1. 587 /// `inner` always has a reference count of 1.
567 /// 588 ///
568 /// # Examples 589 /// # Examples
569 /// 590 ///
570 /// In the following example, we make changes 591 /// In the following example, we make changes to the inner object before turning it into an
571 /// `Arc<Test>` object (after which point, it 592 /// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
572 /// cannot fail. 593 /// cannot fail.
573 /// 594 ///
574 /// ``` 595 /// ```
575 /// use kernel::sync::{Arc, UniqueArc}; 596 /// use kernel::sync::{Arc, UniqueArc};
576 /// 597 ///
577 /// struct Example { 598 /// struct Example {
578 /// a: u32, 599 /// a: u32,
579 /// b: u32, 600 /// b: u32,
580 /// } 601 /// }
581 /// 602 ///
582 /// fn test() -> Result<Arc<Example>> { 603 /// fn test() -> Result<Arc<Example>> {
583 /// let mut x = UniqueArc::new(Example { a 604 /// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?;
584 /// x.a += 1; 605 /// x.a += 1;
585 /// x.b += 1; 606 /// x.b += 1;
586 /// Ok(x.into()) 607 /// Ok(x.into())
587 /// } 608 /// }
588 /// 609 ///
589 /// # test().unwrap(); 610 /// # test().unwrap();
590 /// ``` 611 /// ```
591 /// 612 ///
592 /// In the following example we first allocate 613 /// In the following example we first allocate memory for a refcounted `Example` but we don't
593 /// initialise it on allocation. We do initial 614 /// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
594 /// followed by a conversion to `Arc<Example>` 615 /// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
595 /// in one context (e.g., sleepable) and initi 616 /// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
596 /// 617 ///
597 /// ``` 618 /// ```
598 /// use kernel::sync::{Arc, UniqueArc}; 619 /// use kernel::sync::{Arc, UniqueArc};
599 /// 620 ///
600 /// struct Example { 621 /// struct Example {
601 /// a: u32, 622 /// a: u32,
602 /// b: u32, 623 /// b: u32,
603 /// } 624 /// }
604 /// 625 ///
605 /// fn test() -> Result<Arc<Example>> { 626 /// fn test() -> Result<Arc<Example>> {
606 /// let x = UniqueArc::new_uninit(GFP_KERN 627 /// let x = UniqueArc::new_uninit(GFP_KERNEL)?;
607 /// Ok(x.write(Example { a: 10, b: 20 }).i 628 /// Ok(x.write(Example { a: 10, b: 20 }).into())
608 /// } 629 /// }
609 /// 630 ///
610 /// # test().unwrap(); 631 /// # test().unwrap();
611 /// ``` 632 /// ```
612 /// 633 ///
613 /// In the last example below, the caller gets 634 /// In the last example below, the caller gets a pinned instance of `Example` while converting to
614 /// `Arc<Example>`; this is useful in scenario 635 /// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
615 /// initialisation, for example, when initiali 636 /// initialisation, for example, when initialising fields that are wrapped in locks.
616 /// 637 ///
617 /// ``` 638 /// ```
618 /// use kernel::sync::{Arc, UniqueArc}; 639 /// use kernel::sync::{Arc, UniqueArc};
619 /// 640 ///
620 /// struct Example { 641 /// struct Example {
621 /// a: u32, 642 /// a: u32,
622 /// b: u32, 643 /// b: u32,
623 /// } 644 /// }
624 /// 645 ///
625 /// fn test() -> Result<Arc<Example>> { 646 /// fn test() -> Result<Arc<Example>> {
626 /// let mut pinned = Pin::from(UniqueArc:: 647 /// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?);
627 /// // We can modify `pinned` because it i 648 /// // We can modify `pinned` because it is `Unpin`.
628 /// pinned.as_mut().a += 1; 649 /// pinned.as_mut().a += 1;
629 /// Ok(pinned.into()) 650 /// Ok(pinned.into())
630 /// } 651 /// }
631 /// 652 ///
632 /// # test().unwrap(); 653 /// # test().unwrap();
633 /// ``` 654 /// ```
634 pub struct UniqueArc<T: ?Sized> { 655 pub struct UniqueArc<T: ?Sized> {
635 inner: Arc<T>, 656 inner: Arc<T>,
636 } 657 }
637 658
638 impl<T> UniqueArc<T> { 659 impl<T> UniqueArc<T> {
639 /// Tries to allocate a new [`UniqueArc`] 660 /// Tries to allocate a new [`UniqueArc`] instance.
640 pub fn new(value: T, flags: Flags) -> Resu 661 pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> {
641 Ok(Self { 662 Ok(Self {
642 // INVARIANT: The newly-created ob 663 // INVARIANT: The newly-created object has a refcount of 1.
643 inner: Arc::new(value, flags)?, 664 inner: Arc::new(value, flags)?,
644 }) 665 })
645 } 666 }
646 667
647 /// Tries to allocate a new [`UniqueArc`] 668 /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
648 pub fn new_uninit(flags: Flags) -> Result< 669 pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {
649 // INVARIANT: The refcount is initiali 670 // INVARIANT: The refcount is initialised to a non-zero value.
650 let inner = Box::try_init::<AllocError 671 let inner = Box::try_init::<AllocError>(
651 try_init!(ArcInner { 672 try_init!(ArcInner {
652 // SAFETY: There are no safety 673 // SAFETY: There are no safety requirements for this FFI call.
653 refcount: Opaque::new(unsafe { 674 refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
654 data <- init::uninit::<T, Allo 675 data <- init::uninit::<T, AllocError>(),
655 }? AllocError), 676 }? AllocError),
656 flags, 677 flags,
657 )?; 678 )?;
658 Ok(UniqueArc { 679 Ok(UniqueArc {
659 // INVARIANT: The newly-created ob 680 // INVARIANT: The newly-created object has a refcount of 1.
660 // SAFETY: The pointer from the `B 681 // SAFETY: The pointer from the `Box` is valid.
661 inner: unsafe { Arc::from_inner(Bo 682 inner: unsafe { Arc::from_inner(Box::leak(inner).into()) },
662 }) 683 })
663 } 684 }
664 } 685 }
665 686
666 impl<T> UniqueArc<MaybeUninit<T>> { 687 impl<T> UniqueArc<MaybeUninit<T>> {
667 /// Converts a `UniqueArc<MaybeUninit<T>>` 688 /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
668 pub fn write(mut self, value: T) -> Unique 689 pub fn write(mut self, value: T) -> UniqueArc<T> {
669 self.deref_mut().write(value); 690 self.deref_mut().write(value);
670 // SAFETY: We just wrote the value to 691 // SAFETY: We just wrote the value to be initialized.
671 unsafe { self.assume_init() } 692 unsafe { self.assume_init() }
672 } 693 }
673 694
674 /// Unsafely assume that `self` is initial 695 /// Unsafely assume that `self` is initialized.
675 /// 696 ///
676 /// # Safety 697 /// # Safety
677 /// 698 ///
678 /// The caller guarantees that the value b 699 /// The caller guarantees that the value behind this pointer has been initialized. It is
679 /// *immediate* UB to call this when the v 700 /// *immediate* UB to call this when the value is not initialized.
680 pub unsafe fn assume_init(self) -> UniqueA 701 pub unsafe fn assume_init(self) -> UniqueArc<T> {
681 let inner = ManuallyDrop::new(self).in 702 let inner = ManuallyDrop::new(self).inner.ptr;
682 UniqueArc { 703 UniqueArc {
683 // SAFETY: The new `Arc` is taking 704 // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
684 // dropped). The types are compati 705 // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
685 inner: unsafe { Arc::from_inner(in 706 inner: unsafe { Arc::from_inner(inner.cast()) },
686 } 707 }
687 } 708 }
688 709
689 /// Initialize `self` using the given init 710 /// Initialize `self` using the given initializer.
690 pub fn init_with<E>(mut self, init: impl I 711 pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
691 // SAFETY: The supplied pointer is val 712 // SAFETY: The supplied pointer is valid for initialization.
692 match unsafe { init.__init(self.as_mut 713 match unsafe { init.__init(self.as_mut_ptr()) } {
693 // SAFETY: Initialization complete 714 // SAFETY: Initialization completed successfully.
694 Ok(()) => Ok(unsafe { self.assume_ 715 Ok(()) => Ok(unsafe { self.assume_init() }),
695 Err(err) => Err(err), 716 Err(err) => Err(err),
696 } 717 }
697 } 718 }
698 719
699 /// Pin-initialize `self` using the given 720 /// Pin-initialize `self` using the given pin-initializer.
700 pub fn pin_init_with<E>( 721 pub fn pin_init_with<E>(
701 mut self, 722 mut self,
702 init: impl PinInit<T, E>, 723 init: impl PinInit<T, E>,
703 ) -> core::result::Result<Pin<UniqueArc<T> 724 ) -> core::result::Result<Pin<UniqueArc<T>>, E> {
704 // SAFETY: The supplied pointer is val 725 // SAFETY: The supplied pointer is valid for initialization and we will later pin the value
705 // to ensure it does not move. 726 // to ensure it does not move.
706 match unsafe { init.__pinned_init(self 727 match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
707 // SAFETY: Initialization complete 728 // SAFETY: Initialization completed successfully.
708 Ok(()) => Ok(unsafe { self.assume_ 729 Ok(()) => Ok(unsafe { self.assume_init() }.into()),
709 Err(err) => Err(err), 730 Err(err) => Err(err),
710 } 731 }
711 } 732 }
712 } 733 }
713 734
714 impl<T: ?Sized> From<UniqueArc<T>> for Pin<Uni 735 impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
715 fn from(obj: UniqueArc<T>) -> Self { 736 fn from(obj: UniqueArc<T>) -> Self {
716 // SAFETY: It is not possible to move/ 737 // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
717 // is `Unpin`), so it is ok to convert 738 // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
718 unsafe { Pin::new_unchecked(obj) } 739 unsafe { Pin::new_unchecked(obj) }
719 } 740 }
720 } 741 }
721 742
722 impl<T: ?Sized> Deref for UniqueArc<T> { 743 impl<T: ?Sized> Deref for UniqueArc<T> {
723 type Target = T; 744 type Target = T;
724 745
725 fn deref(&self) -> &Self::Target { 746 fn deref(&self) -> &Self::Target {
726 self.inner.deref() 747 self.inner.deref()
727 } 748 }
728 } 749 }
729 750
730 impl<T: ?Sized> DerefMut for UniqueArc<T> { 751 impl<T: ?Sized> DerefMut for UniqueArc<T> {
731 fn deref_mut(&mut self) -> &mut Self::Targ 752 fn deref_mut(&mut self) -> &mut Self::Target {
732 // SAFETY: By the `Arc` type invariant 753 // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
733 // it is safe to dereference it. Addit 754 // it is safe to dereference it. Additionally, we know there is only one reference when
734 // it's inside a `UniqueArc`, so it is 755 // it's inside a `UniqueArc`, so it is safe to get a mutable reference.
735 unsafe { &mut self.inner.ptr.as_mut(). 756 unsafe { &mut self.inner.ptr.as_mut().data }
736 } 757 }
737 } 758 }
738 759
739 impl<T: fmt::Display + ?Sized> fmt::Display fo 760 impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {
740 fn fmt(&self, f: &mut fmt::Formatter<'_>) 761 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
741 fmt::Display::fmt(self.deref(), f) 762 fmt::Display::fmt(self.deref(), f)
742 } 763 }
743 } 764 }
744 765
745 impl<T: fmt::Display + ?Sized> fmt::Display fo 766 impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {
746 fn fmt(&self, f: &mut fmt::Formatter<'_>) 767 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
747 fmt::Display::fmt(self.deref(), f) 768 fmt::Display::fmt(self.deref(), f)
748 } 769 }
749 } 770 }
750 771
751 impl<T: fmt::Debug + ?Sized> fmt::Debug for Un 772 impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {
752 fn fmt(&self, f: &mut fmt::Formatter<'_>) 773 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
753 fmt::Debug::fmt(self.deref(), f) 774 fmt::Debug::fmt(self.deref(), f)
754 } 775 }
755 } 776 }
756 777
757 impl<T: fmt::Debug + ?Sized> fmt::Debug for Ar 778 impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {
758 fn fmt(&self, f: &mut fmt::Formatter<'_>) 779 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
759 fmt::Debug::fmt(self.deref(), f) 780 fmt::Debug::fmt(self.deref(), f)
760 } 781 }
761 } 782 }
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.