1 // SPDX-License-Identifier: GPL-2.0
2
3 //! This module provides an interface for blk-
4 //!
5 //! C header: [`include/linux/blk-mq.h`](srctr
6
7 use crate::{
8 bindings,
9 block::mq::request::RequestDataWrapper,
10 block::mq::Request,
11 error::{from_result, Result},
12 types::ARef,
13 };
14 use core::{marker::PhantomData, sync::atomic::
15
16 /// Implement this trait to interface blk-mq a
17 ///
18 /// To implement a block device driver, implem
19 /// [module level documentation]. The kernel w
20 /// functions defined in this trait to interfa
21 /// There is no need for an exit_request() imp
22 /// implementation of the [`Request`] type wil
23 /// the C/Rust glue logic.
24 ///
25 /// [module level documentation]: kernel::bloc
26 #[macros::vtable]
27 pub trait Operations: Sized {
28 /// Called by the kernel to queue a reques
29 /// `false`, the driver is allowed to defe
30 fn queue_rq(rq: ARef<Request<Self>>, is_la
31
32 /// Called by the kernel to indicate that
33 fn commit_rqs();
34
35 /// Called by the kernel to poll the devic
36 /// used for poll queues.
37 fn poll() -> bool {
38 crate::build_error(crate::error::VTABL
39 }
40 }
41
42 /// A vtable for blk-mq to interact with a blo
43 ///
44 /// A `bindings::blk_mq_ops` vtable is constru
45 /// "C"` functions of this struct, exposed thr
46 ///
47 /// For general documentation of these methods
48 /// documentation related to `struct blk_mq_op
49 /// [`include/linux/blk-mq.h`].
50 ///
51 /// [`include/linux/blk-mq.h`]: srctree/includ
52 pub(crate) struct OperationsVTable<T: Operatio
53
54 impl<T: Operations> OperationsVTable<T> {
55 /// This function is called by the C kerne
56 /// installed in the `blk_mq_ops` vtable f
57 ///
58 /// # Safety
59 ///
60 /// - The caller of this function must ens
61 /// valid for reads for the duration of
62 /// - This function must be called for an
63 /// is, `Self::init_hctx_callback` was c
64 /// `Self::exit_hctx_callback()` was not
65 /// - `(*bd).rq` must point to an initiali
66 /// That is, `Self::init_request_callbac
67 /// `Self::exit_request_callback` was no
68 /// - `(*bd).rq` must be owned by the driv
69 /// promise to not access the request un
70 /// `bindings::blk_mq_end_request` for t
71 unsafe extern "C" fn queue_rq_callback(
72 _hctx: *mut bindings::blk_mq_hw_ctx,
73 bd: *const bindings::blk_mq_queue_data
74 ) -> bindings::blk_status_t {
75 // SAFETY: `bd.rq` is valid as require
76 // this function.
77 let request = unsafe { &*(*bd).rq.cast
78
79 // One refcount for the ARef, one for
80 request.wrapper_ref().refcount().store
81
82 // SAFETY:
83 // - We own a refcount that we took a
84 // - By the safety requirements of th
85 // `struct request` and the private
86 // - `rq` will be alive until `blk_mq
87 // reference counted by `ARef` unti
88 let rq = unsafe { Request::aref_from_r
89
90 // SAFETY: We have exclusive access an
91 unsafe { Request::start_unchecked(&rq)
92
93 let ret = T::queue_rq(
94 rq,
95 // SAFETY: `bd` is valid as requir
96 // this function.
97 unsafe { (*bd).last },
98 );
99
100 if let Err(e) = ret {
101 e.to_blk_status()
102 } else {
103 bindings::BLK_STS_OK as _
104 }
105 }
106
107 /// This function is called by the C kerne
108 /// installed in the `blk_mq_ops` vtable f
109 ///
110 /// # Safety
111 ///
112 /// This function may only be called by bl
113 unsafe extern "C" fn commit_rqs_callback(_
114 T::commit_rqs()
115 }
116
117 /// This function is called by the C kerne
118 /// implemented, and there is no way to ex
119 ///
120 /// # Safety
121 ///
122 /// This function may only be called by bl
123 unsafe extern "C" fn complete_callback(_rq
124
125 /// This function is called by the C kerne
126 /// installed in the `blk_mq_ops` vtable f
127 ///
128 /// # Safety
129 ///
130 /// This function may only be called by bl
131 unsafe extern "C" fn poll_callback(
132 _hctx: *mut bindings::blk_mq_hw_ctx,
133 _iob: *mut bindings::io_comp_batch,
134 ) -> core::ffi::c_int {
135 T::poll().into()
136 }
137
138 /// This function is called by the C kerne
139 /// installed in the `blk_mq_ops` vtable f
140 ///
141 /// # Safety
142 ///
143 /// This function may only be called by bl
144 /// function may only be called once befor
145 /// for the same context.
146 unsafe extern "C" fn init_hctx_callback(
147 _hctx: *mut bindings::blk_mq_hw_ctx,
148 _tagset_data: *mut core::ffi::c_void,
149 _hctx_idx: core::ffi::c_uint,
150 ) -> core::ffi::c_int {
151 from_result(|| Ok(0))
152 }
153
154 /// This function is called by the C kerne
155 /// installed in the `blk_mq_ops` vtable f
156 ///
157 /// # Safety
158 ///
159 /// This function may only be called by bl
160 unsafe extern "C" fn exit_hctx_callback(
161 _hctx: *mut bindings::blk_mq_hw_ctx,
162 _hctx_idx: core::ffi::c_uint,
163 ) {
164 }
165
166 /// This function is called by the C kerne
167 /// installed in the `blk_mq_ops` vtable f
168 ///
169 /// # Safety
170 ///
171 /// - This function may only be called by
172 /// - `_set` must point to an initialized
173 /// - `rq` must point to an initialized `b
174 /// - The allocation pointed to by `rq` mu
175 /// plus the size of `RequestDataWrapper
176 unsafe extern "C" fn init_request_callback
177 _set: *mut bindings::blk_mq_tag_set,
178 rq: *mut bindings::request,
179 _hctx_idx: core::ffi::c_uint,
180 _numa_node: core::ffi::c_uint,
181 ) -> core::ffi::c_int {
182 from_result(|| {
183 // SAFETY: By the safety requireme
184 // to a valid allocation.
185 let pdu = unsafe { Request::wrappe
186
187 // SAFETY: The refcount field is a
188 // it is valid for writes.
189 unsafe { RequestDataWrapper::refco
190
191 Ok(0)
192 })
193 }
194
195 /// This function is called by the C kerne
196 /// installed in the `blk_mq_ops` vtable f
197 ///
198 /// # Safety
199 ///
200 /// - This function may only be called by
201 /// - `_set` must point to an initialized
202 /// - `rq` must point to an initialized an
203 unsafe extern "C" fn exit_request_callback
204 _set: *mut bindings::blk_mq_tag_set,
205 rq: *mut bindings::request,
206 _hctx_idx: core::ffi::c_uint,
207 ) {
208 // SAFETY: The tagset invariants guara
209 // for the request data.
210 let pdu = unsafe { bindings::blk_mq_rq
211
212 // SAFETY: `pdu` is valid for read and
213 unsafe { core::ptr::drop_in_place(pdu)
214 }
215
216 const VTABLE: bindings::blk_mq_ops = bindi
217 queue_rq: Some(Self::queue_rq_callback
218 queue_rqs: None,
219 commit_rqs: Some(Self::commit_rqs_call
220 get_budget: None,
221 put_budget: None,
222 set_rq_budget_token: None,
223 get_rq_budget_token: None,
224 timeout: None,
225 poll: if T::HAS_POLL {
226 Some(Self::poll_callback)
227 } else {
228 None
229 },
230 complete: Some(Self::complete_callback
231 init_hctx: Some(Self::init_hctx_callba
232 exit_hctx: Some(Self::exit_hctx_callba
233 init_request: Some(Self::init_request_
234 exit_request: Some(Self::exit_request_
235 cleanup_rq: None,
236 busy: None,
237 map_queues: None,
238 #[cfg(CONFIG_BLK_DEBUG_FS)]
239 show_rq: None,
240 };
241
242 pub(crate) const fn build() -> &'static bi
243 &Self::VTABLE
244 }
245 }
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.