1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 2
3 //! This module provides an interface for blk- 3 //! This module provides an interface for blk-mq drivers to implement.
4 //! 4 //!
5 //! C header: [`include/linux/blk-mq.h`](srctr 5 //! C header: [`include/linux/blk-mq.h`](srctree/include/linux/blk-mq.h)
6 6
7 use crate::{ 7 use crate::{
8 bindings, 8 bindings,
9 block::mq::request::RequestDataWrapper, 9 block::mq::request::RequestDataWrapper,
10 block::mq::Request, 10 block::mq::Request,
11 error::{from_result, Result}, 11 error::{from_result, Result},
12 types::ARef, 12 types::ARef,
13 }; 13 };
14 use core::{marker::PhantomData, sync::atomic:: 14 use core::{marker::PhantomData, sync::atomic::AtomicU64, sync::atomic::Ordering};
15 15
16 /// Implement this trait to interface blk-mq a 16 /// Implement this trait to interface blk-mq as block devices.
17 /// 17 ///
18 /// To implement a block device driver, implem 18 /// To implement a block device driver, implement this trait as described in the
19 /// [module level documentation]. The kernel w 19 /// [module level documentation]. The kernel will use the implementation of the
20 /// functions defined in this trait to interfa 20 /// functions defined in this trait to interface a block device driver. Note:
21 /// There is no need for an exit_request() imp 21 /// There is no need for an exit_request() implementation, because the `drop`
22 /// implementation of the [`Request`] type wil 22 /// implementation of the [`Request`] type will be invoked by automatically by
23 /// the C/Rust glue logic. 23 /// the C/Rust glue logic.
24 /// 24 ///
25 /// [module level documentation]: kernel::bloc 25 /// [module level documentation]: kernel::block::mq
26 #[macros::vtable] 26 #[macros::vtable]
27 pub trait Operations: Sized { 27 pub trait Operations: Sized {
28 /// Called by the kernel to queue a reques 28 /// Called by the kernel to queue a request with the driver. If `is_last` is
29 /// `false`, the driver is allowed to defe 29 /// `false`, the driver is allowed to defer committing the request.
30 fn queue_rq(rq: ARef<Request<Self>>, is_la 30 fn queue_rq(rq: ARef<Request<Self>>, is_last: bool) -> Result;
31 31
32 /// Called by the kernel to indicate that 32 /// Called by the kernel to indicate that queued requests should be submitted.
33 fn commit_rqs(); 33 fn commit_rqs();
34 34
35 /// Called by the kernel to poll the devic 35 /// Called by the kernel to poll the device for completed requests. Only
36 /// used for poll queues. 36 /// used for poll queues.
37 fn poll() -> bool { 37 fn poll() -> bool {
38 crate::build_error(crate::error::VTABL 38 crate::build_error(crate::error::VTABLE_DEFAULT_ERROR)
39 } 39 }
40 } 40 }
41 41
42 /// A vtable for blk-mq to interact with a blo 42 /// A vtable for blk-mq to interact with a block device driver.
43 /// 43 ///
44 /// A `bindings::blk_mq_ops` vtable is constru 44 /// A `bindings::blk_mq_ops` vtable is constructed from pointers to the `extern
45 /// "C"` functions of this struct, exposed thr 45 /// "C"` functions of this struct, exposed through the `OperationsVTable::VTABLE`.
46 /// 46 ///
47 /// For general documentation of these methods 47 /// For general documentation of these methods, see the kernel source
48 /// documentation related to `struct blk_mq_op 48 /// documentation related to `struct blk_mq_operations` in
49 /// [`include/linux/blk-mq.h`]. 49 /// [`include/linux/blk-mq.h`].
50 /// 50 ///
51 /// [`include/linux/blk-mq.h`]: srctree/includ 51 /// [`include/linux/blk-mq.h`]: srctree/include/linux/blk-mq.h
52 pub(crate) struct OperationsVTable<T: Operatio 52 pub(crate) struct OperationsVTable<T: Operations>(PhantomData<T>);
53 53
54 impl<T: Operations> OperationsVTable<T> { 54 impl<T: Operations> OperationsVTable<T> {
55 /// This function is called by the C kerne 55 /// This function is called by the C kernel. A pointer to this function is
56 /// installed in the `blk_mq_ops` vtable f 56 /// installed in the `blk_mq_ops` vtable for the driver.
57 /// 57 ///
58 /// # Safety 58 /// # Safety
59 /// 59 ///
60 /// - The caller of this function must ens 60 /// - The caller of this function must ensure that the pointee of `bd` is
61 /// valid for reads for the duration of 61 /// valid for reads for the duration of this function.
62 /// - This function must be called for an 62 /// - This function must be called for an initialized and live `hctx`. That
63 /// is, `Self::init_hctx_callback` was c 63 /// is, `Self::init_hctx_callback` was called and
64 /// `Self::exit_hctx_callback()` was not 64 /// `Self::exit_hctx_callback()` was not yet called.
65 /// - `(*bd).rq` must point to an initiali 65 /// - `(*bd).rq` must point to an initialized and live `bindings:request`.
66 /// That is, `Self::init_request_callbac 66 /// That is, `Self::init_request_callback` was called but
67 /// `Self::exit_request_callback` was no 67 /// `Self::exit_request_callback` was not yet called for the request.
68 /// - `(*bd).rq` must be owned by the driv 68 /// - `(*bd).rq` must be owned by the driver. That is, the block layer must
69 /// promise to not access the request un 69 /// promise to not access the request until the driver calls
70 /// `bindings::blk_mq_end_request` for t 70 /// `bindings::blk_mq_end_request` for the request.
71 unsafe extern "C" fn queue_rq_callback( 71 unsafe extern "C" fn queue_rq_callback(
72 _hctx: *mut bindings::blk_mq_hw_ctx, 72 _hctx: *mut bindings::blk_mq_hw_ctx,
73 bd: *const bindings::blk_mq_queue_data 73 bd: *const bindings::blk_mq_queue_data,
74 ) -> bindings::blk_status_t { 74 ) -> bindings::blk_status_t {
75 // SAFETY: `bd.rq` is valid as require 75 // SAFETY: `bd.rq` is valid as required by the safety requirement for
76 // this function. 76 // this function.
77 let request = unsafe { &*(*bd).rq.cast 77 let request = unsafe { &*(*bd).rq.cast::<Request<T>>() };
78 78
79 // One refcount for the ARef, one for 79 // One refcount for the ARef, one for being in flight
80 request.wrapper_ref().refcount().store 80 request.wrapper_ref().refcount().store(2, Ordering::Relaxed);
81 81
82 // SAFETY: 82 // SAFETY:
83 // - We own a refcount that we took a 83 // - We own a refcount that we took above. We pass that to `ARef`.
84 // - By the safety requirements of th 84 // - By the safety requirements of this function, `request` is a valid
85 // `struct request` and the private 85 // `struct request` and the private data is properly initialized.
86 // - `rq` will be alive until `blk_mq 86 // - `rq` will be alive until `blk_mq_end_request` is called and is
87 // reference counted by `ARef` unti 87 // reference counted by `ARef` until then.
88 let rq = unsafe { Request::aref_from_r 88 let rq = unsafe { Request::aref_from_raw((*bd).rq) };
89 89
90 // SAFETY: We have exclusive access an 90 // SAFETY: We have exclusive access and we just set the refcount above.
91 unsafe { Request::start_unchecked(&rq) 91 unsafe { Request::start_unchecked(&rq) };
92 92
93 let ret = T::queue_rq( 93 let ret = T::queue_rq(
94 rq, 94 rq,
95 // SAFETY: `bd` is valid as requir 95 // SAFETY: `bd` is valid as required by the safety requirement for
96 // this function. 96 // this function.
97 unsafe { (*bd).last }, 97 unsafe { (*bd).last },
98 ); 98 );
99 99
100 if let Err(e) = ret { 100 if let Err(e) = ret {
101 e.to_blk_status() 101 e.to_blk_status()
102 } else { 102 } else {
103 bindings::BLK_STS_OK as _ 103 bindings::BLK_STS_OK as _
104 } 104 }
105 } 105 }
106 106
107 /// This function is called by the C kerne 107 /// This function is called by the C kernel. A pointer to this function is
108 /// installed in the `blk_mq_ops` vtable f 108 /// installed in the `blk_mq_ops` vtable for the driver.
109 /// 109 ///
110 /// # Safety 110 /// # Safety
111 /// 111 ///
112 /// This function may only be called by bl 112 /// This function may only be called by blk-mq C infrastructure.
113 unsafe extern "C" fn commit_rqs_callback(_ 113 unsafe extern "C" fn commit_rqs_callback(_hctx: *mut bindings::blk_mq_hw_ctx) {
114 T::commit_rqs() 114 T::commit_rqs()
115 } 115 }
116 116
117 /// This function is called by the C kerne 117 /// This function is called by the C kernel. It is not currently
118 /// implemented, and there is no way to ex 118 /// implemented, and there is no way to exercise this code path.
119 /// 119 ///
120 /// # Safety 120 /// # Safety
121 /// 121 ///
122 /// This function may only be called by bl 122 /// This function may only be called by blk-mq C infrastructure.
123 unsafe extern "C" fn complete_callback(_rq 123 unsafe extern "C" fn complete_callback(_rq: *mut bindings::request) {}
124 124
125 /// This function is called by the C kerne 125 /// This function is called by the C kernel. A pointer to this function is
126 /// installed in the `blk_mq_ops` vtable f 126 /// installed in the `blk_mq_ops` vtable for the driver.
127 /// 127 ///
128 /// # Safety 128 /// # Safety
129 /// 129 ///
130 /// This function may only be called by bl 130 /// This function may only be called by blk-mq C infrastructure.
131 unsafe extern "C" fn poll_callback( 131 unsafe extern "C" fn poll_callback(
132 _hctx: *mut bindings::blk_mq_hw_ctx, 132 _hctx: *mut bindings::blk_mq_hw_ctx,
133 _iob: *mut bindings::io_comp_batch, 133 _iob: *mut bindings::io_comp_batch,
134 ) -> core::ffi::c_int { 134 ) -> core::ffi::c_int {
135 T::poll().into() 135 T::poll().into()
136 } 136 }
137 137
138 /// This function is called by the C kerne 138 /// This function is called by the C kernel. A pointer to this function is
139 /// installed in the `blk_mq_ops` vtable f 139 /// installed in the `blk_mq_ops` vtable for the driver.
140 /// 140 ///
141 /// # Safety 141 /// # Safety
142 /// 142 ///
143 /// This function may only be called by bl 143 /// This function may only be called by blk-mq C infrastructure. This
144 /// function may only be called once befor 144 /// function may only be called once before `exit_hctx_callback` is called
145 /// for the same context. 145 /// for the same context.
146 unsafe extern "C" fn init_hctx_callback( 146 unsafe extern "C" fn init_hctx_callback(
147 _hctx: *mut bindings::blk_mq_hw_ctx, 147 _hctx: *mut bindings::blk_mq_hw_ctx,
148 _tagset_data: *mut core::ffi::c_void, 148 _tagset_data: *mut core::ffi::c_void,
149 _hctx_idx: core::ffi::c_uint, 149 _hctx_idx: core::ffi::c_uint,
150 ) -> core::ffi::c_int { 150 ) -> core::ffi::c_int {
151 from_result(|| Ok(0)) 151 from_result(|| Ok(0))
152 } 152 }
153 153
154 /// This function is called by the C kerne 154 /// This function is called by the C kernel. A pointer to this function is
155 /// installed in the `blk_mq_ops` vtable f 155 /// installed in the `blk_mq_ops` vtable for the driver.
156 /// 156 ///
157 /// # Safety 157 /// # Safety
158 /// 158 ///
159 /// This function may only be called by bl 159 /// This function may only be called by blk-mq C infrastructure.
160 unsafe extern "C" fn exit_hctx_callback( 160 unsafe extern "C" fn exit_hctx_callback(
161 _hctx: *mut bindings::blk_mq_hw_ctx, 161 _hctx: *mut bindings::blk_mq_hw_ctx,
162 _hctx_idx: core::ffi::c_uint, 162 _hctx_idx: core::ffi::c_uint,
163 ) { 163 ) {
164 } 164 }
165 165
166 /// This function is called by the C kerne 166 /// This function is called by the C kernel. A pointer to this function is
167 /// installed in the `blk_mq_ops` vtable f 167 /// installed in the `blk_mq_ops` vtable for the driver.
168 /// 168 ///
169 /// # Safety 169 /// # Safety
170 /// 170 ///
171 /// - This function may only be called by 171 /// - This function may only be called by blk-mq C infrastructure.
172 /// - `_set` must point to an initialized 172 /// - `_set` must point to an initialized `TagSet<T>`.
173 /// - `rq` must point to an initialized `b 173 /// - `rq` must point to an initialized `bindings::request`.
174 /// - The allocation pointed to by `rq` mu 174 /// - The allocation pointed to by `rq` must be at the size of `Request`
175 /// plus the size of `RequestDataWrapper 175 /// plus the size of `RequestDataWrapper`.
176 unsafe extern "C" fn init_request_callback 176 unsafe extern "C" fn init_request_callback(
177 _set: *mut bindings::blk_mq_tag_set, 177 _set: *mut bindings::blk_mq_tag_set,
178 rq: *mut bindings::request, 178 rq: *mut bindings::request,
179 _hctx_idx: core::ffi::c_uint, 179 _hctx_idx: core::ffi::c_uint,
180 _numa_node: core::ffi::c_uint, 180 _numa_node: core::ffi::c_uint,
181 ) -> core::ffi::c_int { 181 ) -> core::ffi::c_int {
182 from_result(|| { 182 from_result(|| {
183 // SAFETY: By the safety requireme 183 // SAFETY: By the safety requirements of this function, `rq` points
184 // to a valid allocation. 184 // to a valid allocation.
185 let pdu = unsafe { Request::wrappe 185 let pdu = unsafe { Request::wrapper_ptr(rq.cast::<Request<T>>()) };
186 186
187 // SAFETY: The refcount field is a 187 // SAFETY: The refcount field is allocated but not initialized, so
188 // it is valid for writes. 188 // it is valid for writes.
189 unsafe { RequestDataWrapper::refco 189 unsafe { RequestDataWrapper::refcount_ptr(pdu.as_ptr()).write(AtomicU64::new(0)) };
190 190
191 Ok(0) 191 Ok(0)
192 }) 192 })
193 } 193 }
194 194
195 /// This function is called by the C kerne 195 /// This function is called by the C kernel. A pointer to this function is
196 /// installed in the `blk_mq_ops` vtable f 196 /// installed in the `blk_mq_ops` vtable for the driver.
197 /// 197 ///
198 /// # Safety 198 /// # Safety
199 /// 199 ///
200 /// - This function may only be called by 200 /// - This function may only be called by blk-mq C infrastructure.
201 /// - `_set` must point to an initialized 201 /// - `_set` must point to an initialized `TagSet<T>`.
202 /// - `rq` must point to an initialized an 202 /// - `rq` must point to an initialized and valid `Request`.
203 unsafe extern "C" fn exit_request_callback 203 unsafe extern "C" fn exit_request_callback(
204 _set: *mut bindings::blk_mq_tag_set, 204 _set: *mut bindings::blk_mq_tag_set,
205 rq: *mut bindings::request, 205 rq: *mut bindings::request,
206 _hctx_idx: core::ffi::c_uint, 206 _hctx_idx: core::ffi::c_uint,
207 ) { 207 ) {
208 // SAFETY: The tagset invariants guara 208 // SAFETY: The tagset invariants guarantee that all requests are allocated with extra memory
209 // for the request data. 209 // for the request data.
210 let pdu = unsafe { bindings::blk_mq_rq 210 let pdu = unsafe { bindings::blk_mq_rq_to_pdu(rq) }.cast::<RequestDataWrapper>();
211 211
212 // SAFETY: `pdu` is valid for read and 212 // SAFETY: `pdu` is valid for read and write and is properly initialised.
213 unsafe { core::ptr::drop_in_place(pdu) 213 unsafe { core::ptr::drop_in_place(pdu) };
214 } 214 }
215 215
216 const VTABLE: bindings::blk_mq_ops = bindi 216 const VTABLE: bindings::blk_mq_ops = bindings::blk_mq_ops {
217 queue_rq: Some(Self::queue_rq_callback 217 queue_rq: Some(Self::queue_rq_callback),
218 queue_rqs: None, 218 queue_rqs: None,
219 commit_rqs: Some(Self::commit_rqs_call 219 commit_rqs: Some(Self::commit_rqs_callback),
220 get_budget: None, 220 get_budget: None,
221 put_budget: None, 221 put_budget: None,
222 set_rq_budget_token: None, 222 set_rq_budget_token: None,
223 get_rq_budget_token: None, 223 get_rq_budget_token: None,
224 timeout: None, 224 timeout: None,
225 poll: if T::HAS_POLL { 225 poll: if T::HAS_POLL {
226 Some(Self::poll_callback) 226 Some(Self::poll_callback)
227 } else { 227 } else {
228 None 228 None
229 }, 229 },
230 complete: Some(Self::complete_callback 230 complete: Some(Self::complete_callback),
231 init_hctx: Some(Self::init_hctx_callba 231 init_hctx: Some(Self::init_hctx_callback),
232 exit_hctx: Some(Self::exit_hctx_callba 232 exit_hctx: Some(Self::exit_hctx_callback),
233 init_request: Some(Self::init_request_ 233 init_request: Some(Self::init_request_callback),
234 exit_request: Some(Self::exit_request_ 234 exit_request: Some(Self::exit_request_callback),
235 cleanup_rq: None, 235 cleanup_rq: None,
236 busy: None, 236 busy: None,
237 map_queues: None, 237 map_queues: None,
238 #[cfg(CONFIG_BLK_DEBUG_FS)] 238 #[cfg(CONFIG_BLK_DEBUG_FS)]
239 show_rq: None, 239 show_rq: None,
240 }; 240 };
241 241
242 pub(crate) const fn build() -> &'static bi 242 pub(crate) const fn build() -> &'static bindings::blk_mq_ops {
243 &Self::VTABLE 243 &Self::VTABLE
244 } 244 }
245 } 245 }
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.