1 /* SPDX-License-Identifier: GPL-2.0-or-later * 1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* 2 /*
3 * Copyright (C) 2017, Microsoft Corporation 3 * Copyright (C) 2017, Microsoft Corporation.
4 * 4 *
5 * Author(s): Long Li <longli@microsoft.com> 5 * Author(s): Long Li <longli@microsoft.com>
6 */ 6 */
7 #ifndef _SMBDIRECT_H 7 #ifndef _SMBDIRECT_H
8 #define _SMBDIRECT_H 8 #define _SMBDIRECT_H
9 9
10 #ifdef CONFIG_CIFS_SMB_DIRECT 10 #ifdef CONFIG_CIFS_SMB_DIRECT
11 #define cifs_rdma_enabled(server) ((serv 11 #define cifs_rdma_enabled(server) ((server)->rdma)
12 12
13 #include "cifsglob.h" 13 #include "cifsglob.h"
14 #include <rdma/ib_verbs.h> 14 #include <rdma/ib_verbs.h>
15 #include <rdma/rdma_cm.h> 15 #include <rdma/rdma_cm.h>
16 #include <linux/mempool.h> 16 #include <linux/mempool.h>
17 17
18 extern int rdma_readwrite_threshold; 18 extern int rdma_readwrite_threshold;
19 extern int smbd_max_frmr_depth; 19 extern int smbd_max_frmr_depth;
20 extern int smbd_keep_alive_interval; 20 extern int smbd_keep_alive_interval;
21 extern int smbd_max_receive_size; 21 extern int smbd_max_receive_size;
22 extern int smbd_max_fragmented_recv_size; 22 extern int smbd_max_fragmented_recv_size;
23 extern int smbd_max_send_size; 23 extern int smbd_max_send_size;
24 extern int smbd_send_credit_target; 24 extern int smbd_send_credit_target;
25 extern int smbd_receive_credit_max; 25 extern int smbd_receive_credit_max;
26 26
27 enum keep_alive_status { 27 enum keep_alive_status {
28 KEEP_ALIVE_NONE, 28 KEEP_ALIVE_NONE,
29 KEEP_ALIVE_PENDING, 29 KEEP_ALIVE_PENDING,
30 KEEP_ALIVE_SENT, 30 KEEP_ALIVE_SENT,
31 }; 31 };
32 32
33 enum smbd_connection_status { 33 enum smbd_connection_status {
34 SMBD_CREATED, 34 SMBD_CREATED,
35 SMBD_CONNECTING, 35 SMBD_CONNECTING,
36 SMBD_CONNECTED, 36 SMBD_CONNECTED,
37 SMBD_NEGOTIATE_FAILED, 37 SMBD_NEGOTIATE_FAILED,
38 SMBD_DISCONNECTING, 38 SMBD_DISCONNECTING,
39 SMBD_DISCONNECTED, 39 SMBD_DISCONNECTED,
40 SMBD_DESTROYED 40 SMBD_DESTROYED
41 }; 41 };
42 42
43 /* 43 /*
44 * The context for the SMBDirect transport 44 * The context for the SMBDirect transport
45 * Everything related to the transport is here 45 * Everything related to the transport is here. It has several logical parts
46 * 1. RDMA related structures 46 * 1. RDMA related structures
47 * 2. SMBDirect connection parameters 47 * 2. SMBDirect connection parameters
48 * 3. Memory registrations 48 * 3. Memory registrations
49 * 4. Receive and reassembly queues for data r 49 * 4. Receive and reassembly queues for data receive path
50 * 5. mempools for allocating packets 50 * 5. mempools for allocating packets
51 */ 51 */
52 struct smbd_connection { 52 struct smbd_connection {
53 enum smbd_connection_status transport_ 53 enum smbd_connection_status transport_status;
54 54
55 /* RDMA related */ 55 /* RDMA related */
56 struct rdma_cm_id *id; 56 struct rdma_cm_id *id;
57 struct ib_qp_init_attr qp_attr; 57 struct ib_qp_init_attr qp_attr;
58 struct ib_pd *pd; 58 struct ib_pd *pd;
59 struct ib_cq *send_cq, *recv_cq; 59 struct ib_cq *send_cq, *recv_cq;
60 struct ib_device_attr dev_attr; 60 struct ib_device_attr dev_attr;
61 int ri_rc; 61 int ri_rc;
62 struct completion ri_done; 62 struct completion ri_done;
63 wait_queue_head_t conn_wait; 63 wait_queue_head_t conn_wait;
64 wait_queue_head_t disconn_wait; 64 wait_queue_head_t disconn_wait;
65 65
66 struct completion negotiate_completion 66 struct completion negotiate_completion;
67 bool negotiate_done; 67 bool negotiate_done;
68 68
69 struct work_struct disconnect_work; 69 struct work_struct disconnect_work;
70 struct work_struct post_send_credits_w 70 struct work_struct post_send_credits_work;
71 71
72 spinlock_t lock_new_credits_offered; 72 spinlock_t lock_new_credits_offered;
73 int new_credits_offered; 73 int new_credits_offered;
74 74
75 /* Connection parameters defined in [M 75 /* Connection parameters defined in [MS-SMBD] 3.1.1.1 */
76 int receive_credit_max; 76 int receive_credit_max;
77 int send_credit_target; 77 int send_credit_target;
78 int max_send_size; 78 int max_send_size;
79 int max_fragmented_recv_size; 79 int max_fragmented_recv_size;
80 int max_fragmented_send_size; 80 int max_fragmented_send_size;
81 int max_receive_size; 81 int max_receive_size;
82 int keep_alive_interval; 82 int keep_alive_interval;
83 int max_readwrite_size; 83 int max_readwrite_size;
84 enum keep_alive_status keep_alive_requ 84 enum keep_alive_status keep_alive_requested;
85 int protocol; 85 int protocol;
86 atomic_t send_credits; 86 atomic_t send_credits;
87 atomic_t receive_credits; 87 atomic_t receive_credits;
88 int receive_credit_target; 88 int receive_credit_target;
89 int fragment_reassembly_remaining; 89 int fragment_reassembly_remaining;
90 90
91 /* Memory registrations */ 91 /* Memory registrations */
92 /* Maximum number of RDMA read/write o 92 /* Maximum number of RDMA read/write outstanding on this connection */
93 int responder_resources; 93 int responder_resources;
94 /* Maximum number of pages in a single 94 /* Maximum number of pages in a single RDMA write/read on this connection */
95 int max_frmr_depth; 95 int max_frmr_depth;
96 /* 96 /*
97 * If payload is less than or equal to 97 * If payload is less than or equal to the threshold,
98 * use RDMA send/recv to send upper la 98 * use RDMA send/recv to send upper layer I/O.
99 * If payload is more than the thresho 99 * If payload is more than the threshold,
100 * use RDMA read/write through memory 100 * use RDMA read/write through memory registration for I/O.
101 */ 101 */
102 int rdma_readwrite_threshold; 102 int rdma_readwrite_threshold;
103 enum ib_mr_type mr_type; 103 enum ib_mr_type mr_type;
104 struct list_head mr_list; 104 struct list_head mr_list;
105 spinlock_t mr_list_lock; 105 spinlock_t mr_list_lock;
106 /* The number of available MRs ready f 106 /* The number of available MRs ready for memory registration */
107 atomic_t mr_ready_count; 107 atomic_t mr_ready_count;
108 atomic_t mr_used_count; 108 atomic_t mr_used_count;
109 wait_queue_head_t wait_mr; 109 wait_queue_head_t wait_mr;
110 struct work_struct mr_recovery_work; 110 struct work_struct mr_recovery_work;
111 /* Used by transport to wait until all 111 /* Used by transport to wait until all MRs are returned */
112 wait_queue_head_t wait_for_mr_cleanup; 112 wait_queue_head_t wait_for_mr_cleanup;
113 113
114 /* Activity accounting */ 114 /* Activity accounting */
115 atomic_t send_pending; 115 atomic_t send_pending;
116 wait_queue_head_t wait_send_pending; 116 wait_queue_head_t wait_send_pending;
117 wait_queue_head_t wait_post_send; 117 wait_queue_head_t wait_post_send;
118 118
119 /* Receive queue */ 119 /* Receive queue */
120 struct list_head receive_queue; 120 struct list_head receive_queue;
121 int count_receive_queue; 121 int count_receive_queue;
122 spinlock_t receive_queue_lock; 122 spinlock_t receive_queue_lock;
123 123
124 struct list_head empty_packet_queue; 124 struct list_head empty_packet_queue;
125 int count_empty_packet_queue; 125 int count_empty_packet_queue;
126 spinlock_t empty_packet_queue_lock; 126 spinlock_t empty_packet_queue_lock;
127 127
128 wait_queue_head_t wait_receive_queues; 128 wait_queue_head_t wait_receive_queues;
129 129
130 /* Reassembly queue */ 130 /* Reassembly queue */
131 struct list_head reassembly_queue; 131 struct list_head reassembly_queue;
132 spinlock_t reassembly_queue_lock; 132 spinlock_t reassembly_queue_lock;
133 wait_queue_head_t wait_reassembly_queu 133 wait_queue_head_t wait_reassembly_queue;
134 134
135 /* total data length of reassembly que 135 /* total data length of reassembly queue */
136 int reassembly_data_length; 136 int reassembly_data_length;
137 int reassembly_queue_length; 137 int reassembly_queue_length;
138 /* the offset to first buffer in reass 138 /* the offset to first buffer in reassembly queue */
139 int first_entry_offset; 139 int first_entry_offset;
140 140
141 bool send_immediate; 141 bool send_immediate;
142 142
143 wait_queue_head_t wait_send_queue; 143 wait_queue_head_t wait_send_queue;
144 144
145 /* 145 /*
146 * Indicate if we have received a full 146 * Indicate if we have received a full packet on the connection
147 * This is used to identify the first 147 * This is used to identify the first SMBD packet of a assembled
148 * payload (SMB packet) in reassembly 148 * payload (SMB packet) in reassembly queue so we can return a
149 * RFC1002 length to upper layer to in 149 * RFC1002 length to upper layer to indicate the length of the SMB
150 * packet received 150 * packet received
151 */ 151 */
152 bool full_packet_received; 152 bool full_packet_received;
153 153
154 struct workqueue_struct *workqueue; 154 struct workqueue_struct *workqueue;
155 struct delayed_work idle_timer_work; 155 struct delayed_work idle_timer_work;
156 156
157 /* Memory pool for preallocating buffe 157 /* Memory pool for preallocating buffers */
158 /* request pool for RDMA send */ 158 /* request pool for RDMA send */
159 struct kmem_cache *request_cache; 159 struct kmem_cache *request_cache;
160 mempool_t *request_mempool; 160 mempool_t *request_mempool;
161 161
162 /* response pool for RDMA receive */ 162 /* response pool for RDMA receive */
163 struct kmem_cache *response_cache; 163 struct kmem_cache *response_cache;
164 mempool_t *response_mempool; 164 mempool_t *response_mempool;
165 165
166 /* for debug purposes */ 166 /* for debug purposes */
167 unsigned int count_get_receive_buffer; 167 unsigned int count_get_receive_buffer;
168 unsigned int count_put_receive_buffer; 168 unsigned int count_put_receive_buffer;
169 unsigned int count_reassembly_queue; 169 unsigned int count_reassembly_queue;
170 unsigned int count_enqueue_reassembly_ 170 unsigned int count_enqueue_reassembly_queue;
171 unsigned int count_dequeue_reassembly_ 171 unsigned int count_dequeue_reassembly_queue;
172 unsigned int count_send_empty; 172 unsigned int count_send_empty;
173 }; 173 };
174 174
175 enum smbd_message_type { 175 enum smbd_message_type {
176 SMBD_NEGOTIATE_RESP, 176 SMBD_NEGOTIATE_RESP,
177 SMBD_TRANSFER_DATA, 177 SMBD_TRANSFER_DATA,
178 }; 178 };
179 179
180 #define SMB_DIRECT_RESPONSE_REQUESTED 0x0001 180 #define SMB_DIRECT_RESPONSE_REQUESTED 0x0001
181 181
182 /* SMBD negotiation request packet [MS-SMBD] 2 182 /* SMBD negotiation request packet [MS-SMBD] 2.2.1 */
183 struct smbd_negotiate_req { 183 struct smbd_negotiate_req {
184 __le16 min_version; 184 __le16 min_version;
185 __le16 max_version; 185 __le16 max_version;
186 __le16 reserved; 186 __le16 reserved;
187 __le16 credits_requested; 187 __le16 credits_requested;
188 __le32 preferred_send_size; 188 __le32 preferred_send_size;
189 __le32 max_receive_size; 189 __le32 max_receive_size;
190 __le32 max_fragmented_size; 190 __le32 max_fragmented_size;
191 } __packed; 191 } __packed;
192 192
193 /* SMBD negotiation response packet [MS-SMBD] 193 /* SMBD negotiation response packet [MS-SMBD] 2.2.2 */
194 struct smbd_negotiate_resp { 194 struct smbd_negotiate_resp {
195 __le16 min_version; 195 __le16 min_version;
196 __le16 max_version; 196 __le16 max_version;
197 __le16 negotiated_version; 197 __le16 negotiated_version;
198 __le16 reserved; 198 __le16 reserved;
199 __le16 credits_requested; 199 __le16 credits_requested;
200 __le16 credits_granted; 200 __le16 credits_granted;
201 __le32 status; 201 __le32 status;
202 __le32 max_readwrite_size; 202 __le32 max_readwrite_size;
203 __le32 preferred_send_size; 203 __le32 preferred_send_size;
204 __le32 max_receive_size; 204 __le32 max_receive_size;
205 __le32 max_fragmented_size; 205 __le32 max_fragmented_size;
206 } __packed; 206 } __packed;
207 207
208 /* SMBD data transfer packet with payload [MS- 208 /* SMBD data transfer packet with payload [MS-SMBD] 2.2.3 */
209 struct smbd_data_transfer { 209 struct smbd_data_transfer {
210 __le16 credits_requested; 210 __le16 credits_requested;
211 __le16 credits_granted; 211 __le16 credits_granted;
212 __le16 flags; 212 __le16 flags;
213 __le16 reserved; 213 __le16 reserved;
214 __le32 remaining_data_length; 214 __le32 remaining_data_length;
215 __le32 data_offset; 215 __le32 data_offset;
216 __le32 data_length; 216 __le32 data_length;
217 __le32 padding; 217 __le32 padding;
218 __u8 buffer[]; 218 __u8 buffer[];
219 } __packed; 219 } __packed;
220 220
221 /* The packet fields for a registered RDMA buf 221 /* The packet fields for a registered RDMA buffer */
222 struct smbd_buffer_descriptor_v1 { 222 struct smbd_buffer_descriptor_v1 {
223 __le64 offset; 223 __le64 offset;
224 __le32 token; 224 __le32 token;
225 __le32 length; 225 __le32 length;
226 } __packed; 226 } __packed;
227 227
228 /* Maximum number of SGEs used by smbdirect.c 228 /* Maximum number of SGEs used by smbdirect.c in any send work request */
229 #define SMBDIRECT_MAX_SEND_SGE 6 229 #define SMBDIRECT_MAX_SEND_SGE 6
230 230
231 /* The context for a SMBD request */ 231 /* The context for a SMBD request */
232 struct smbd_request { 232 struct smbd_request {
233 struct smbd_connection *info; 233 struct smbd_connection *info;
234 struct ib_cqe cqe; 234 struct ib_cqe cqe;
235 235
236 /* the SGE entries for this work reque 236 /* the SGE entries for this work request */
237 struct ib_sge sge[SMBDIRECT_MAX_SEND_S 237 struct ib_sge sge[SMBDIRECT_MAX_SEND_SGE];
238 int num_sge; 238 int num_sge;
239 239
240 /* SMBD packet header follows this str 240 /* SMBD packet header follows this structure */
241 u8 packet[]; 241 u8 packet[];
242 }; 242 };
243 243
244 /* Maximum number of SGEs used by smbdirect.c 244 /* Maximum number of SGEs used by smbdirect.c in any receive work request */
245 #define SMBDIRECT_MAX_RECV_SGE 1 245 #define SMBDIRECT_MAX_RECV_SGE 1
246 246
247 /* The context for a SMBD response */ 247 /* The context for a SMBD response */
248 struct smbd_response { 248 struct smbd_response {
249 struct smbd_connection *info; 249 struct smbd_connection *info;
250 struct ib_cqe cqe; 250 struct ib_cqe cqe;
251 struct ib_sge sge; 251 struct ib_sge sge;
252 252
253 enum smbd_message_type type; 253 enum smbd_message_type type;
254 254
255 /* Link to receive queue or reassembly 255 /* Link to receive queue or reassembly queue */
256 struct list_head list; 256 struct list_head list;
257 257
258 /* Indicate if this is the 1st packet 258 /* Indicate if this is the 1st packet of a payload */
259 bool first_segment; 259 bool first_segment;
260 260
261 /* SMBD packet header and payload foll 261 /* SMBD packet header and payload follows this structure */
262 u8 packet[]; 262 u8 packet[];
263 }; 263 };
264 264
265 /* Create a SMBDirect session */ 265 /* Create a SMBDirect session */
266 struct smbd_connection *smbd_get_connection( 266 struct smbd_connection *smbd_get_connection(
267 struct TCP_Server_Info *server, struct 267 struct TCP_Server_Info *server, struct sockaddr *dstaddr);
268 268
269 /* Reconnect SMBDirect session */ 269 /* Reconnect SMBDirect session */
270 int smbd_reconnect(struct TCP_Server_Info *ser 270 int smbd_reconnect(struct TCP_Server_Info *server);
271 /* Destroy SMBDirect session */ 271 /* Destroy SMBDirect session */
272 void smbd_destroy(struct TCP_Server_Info *serv 272 void smbd_destroy(struct TCP_Server_Info *server);
273 273
274 /* Interface for carrying upper layer I/O thro 274 /* Interface for carrying upper layer I/O through send/recv */
275 int smbd_recv(struct smbd_connection *info, st 275 int smbd_recv(struct smbd_connection *info, struct msghdr *msg);
276 int smbd_send(struct TCP_Server_Info *server, 276 int smbd_send(struct TCP_Server_Info *server,
277 int num_rqst, struct smb_rqst *rqst); 277 int num_rqst, struct smb_rqst *rqst);
278 278
279 enum mr_state { 279 enum mr_state {
280 MR_READY, 280 MR_READY,
281 MR_REGISTERED, 281 MR_REGISTERED,
282 MR_INVALIDATED, 282 MR_INVALIDATED,
283 MR_ERROR 283 MR_ERROR
284 }; 284 };
285 285
286 struct smbd_mr { 286 struct smbd_mr {
287 struct smbd_connection *conn; 287 struct smbd_connection *conn;
288 struct list_head list; 288 struct list_head list;
289 enum mr_state state; 289 enum mr_state state;
290 struct ib_mr *mr; 290 struct ib_mr *mr;
291 struct sg_table sgt; 291 struct sg_table sgt;
292 enum dma_data_direction dir; 292 enum dma_data_direction dir;
293 union { 293 union {
294 struct ib_reg_wr wr; 294 struct ib_reg_wr wr;
295 struct ib_send_wr inv_wr 295 struct ib_send_wr inv_wr;
296 }; 296 };
297 struct ib_cqe cqe; 297 struct ib_cqe cqe;
298 bool need_invalidat 298 bool need_invalidate;
299 struct completion invalidate_don 299 struct completion invalidate_done;
300 }; 300 };
301 301
302 /* Interfaces to register and deregister MR fo 302 /* Interfaces to register and deregister MR for RDMA read/write */
303 struct smbd_mr *smbd_register_mr( 303 struct smbd_mr *smbd_register_mr(
304 struct smbd_connection *info, struct i 304 struct smbd_connection *info, struct iov_iter *iter,
305 bool writing, bool need_invalidate); 305 bool writing, bool need_invalidate);
306 int smbd_deregister_mr(struct smbd_mr *mr); 306 int smbd_deregister_mr(struct smbd_mr *mr);
307 307
308 #else 308 #else
309 #define cifs_rdma_enabled(server) 0 309 #define cifs_rdma_enabled(server) 0
310 struct smbd_connection {}; 310 struct smbd_connection {};
311 static inline void *smbd_get_connection( 311 static inline void *smbd_get_connection(
312 struct TCP_Server_Info *server, struct 312 struct TCP_Server_Info *server, struct sockaddr *dstaddr) {return NULL;}
313 static inline int smbd_reconnect(struct TCP_Se 313 static inline int smbd_reconnect(struct TCP_Server_Info *server) {return -1; }
314 static inline void smbd_destroy(struct TCP_Ser 314 static inline void smbd_destroy(struct TCP_Server_Info *server) {}
315 static inline int smbd_recv(struct smbd_connec 315 static inline int smbd_recv(struct smbd_connection *info, struct msghdr *msg) {return -1; }
316 static inline int smbd_send(struct TCP_Server_ 316 static inline int smbd_send(struct TCP_Server_Info *server, int num_rqst, struct smb_rqst *rqst) {return -1; }
317 #endif 317 #endif
318 318
319 #endif 319 #endif
320 320
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.