1 /* SPDX-License-Identifier: MIT */
2 /* based on linux-kernel/tools/testing/selftests/net/msg_zerocopy.c */
3 #include <assert.h>
4 #include <errno.h>
5 #include <error.h>
6 #include <fcntl.h>
7 #include <limits.h>
8 #include <stdbool.h>
9 #include <stdint.h>
10 #include <stdio.h>
11 #include <stdlib.h>
12 #include <string.h>
13 #include <unistd.h>
14
15 #include <arpa/inet.h>
16 #include <linux/errqueue.h>
17 #include <linux/if_packet.h>
18 #include <linux/io_uring.h>
19 #include <linux/ipv6.h>
20 #include <linux/socket.h>
21 #include <linux/sockios.h>
22 #include <net/ethernet.h>
23 #include <net/if.h>
24 #include <netinet/in.h>
25 #include <netinet/ip.h>
26 #include <netinet/ip6.h>
27 #include <netinet/tcp.h>
28 #include <netinet/udp.h>
29 #include <sys/ioctl.h>
30 #include <sys/mman.h>
31 #include <sys/resource.h>
32 #include <sys/socket.h>
33 #include <sys/stat.h>
34 #include <sys/time.h>
35 #include <sys/types.h>
36 #include <sys/un.h>
37 #include <sys/wait.h>
38
39 #include <io_uring/mini_liburing.h>
40
41 #define NOTIF_TAG 0xfffffffULL
42 #define NONZC_TAG 0
43 #define ZC_TAG 1
44
45 enum {
46 MODE_NONZC = 0,
47 MODE_ZC = 1,
48 MODE_ZC_FIXED = 2,
49 MODE_MIXED = 3,
50 };
51
52 static bool cfg_cork = false;
53 static int cfg_mode = MODE_ZC_FIXED;
54 static int cfg_nr_reqs = 8;
55 static int cfg_family = PF_UNSPEC;
56 static int cfg_payload_len;
57 static int cfg_port = 8000;
58 static int cfg_runtime_ms = 4200;
59
60 static socklen_t cfg_alen;
61 static struct sockaddr_storage cfg_dst_addr;
62
63 static char payload[IP_MAXPACKET] __attribute__((aligned(4096)));
64
65 static unsigned long gettimeofday_ms(void)
66 {
67 struct timeval tv;
68
69 gettimeofday(&tv, NULL);
70 return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
71 }
72
73 static void do_setsockopt(int fd, int level, int optname, int val)
74 {
75 if (setsockopt(fd, level, optname, &val, sizeof(val)))
76 error(1, errno, "setsockopt %d.%d: %d", level, optname, val);
77 }
78
79 static int do_setup_tx(int domain, int type, int protocol)
80 {
81 int fd;
82
83 fd = socket(domain, type, protocol);
84 if (fd == -1)
85 error(1, errno, "socket t");
86
87 do_setsockopt(fd, SOL_SOCKET, SO_SNDBUF, 1 << 21);
88
89 if (connect(fd, (void *) &cfg_dst_addr, cfg_alen))
90 error(1, errno, "connect");
91 return fd;
92 }
93
94 static void do_tx(int domain, int type, int protocol)
95 {
96 struct io_uring_sqe *sqe;
97 struct io_uring_cqe *cqe;
98 unsigned long packets = 0, bytes = 0;
99 struct io_uring ring;
100 struct iovec iov;
101 uint64_t tstop;
102 int i, fd, ret;
103 int compl_cqes = 0;
104
105 fd = do_setup_tx(domain, type, protocol);
106
107 ret = io_uring_queue_init(512, &ring, 0);
108 if (ret)
109 error(1, ret, "io_uring: queue init");
110
111 iov.iov_base = payload;
112 iov.iov_len = cfg_payload_len;
113
114 ret = io_uring_register_buffers(&ring, &iov, 1);
115 if (ret)
116 error(1, ret, "io_uring: buffer registration");
117
118 tstop = gettimeofday_ms() + cfg_runtime_ms;
119 do {
120 if (cfg_cork)
121 do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 1);
122
123 for (i = 0; i < cfg_nr_reqs; i++) {
124 unsigned zc_flags = 0;
125 unsigned buf_idx = 0;
126 unsigned mode = cfg_mode;
127 unsigned msg_flags = MSG_WAITALL;
128
129 if (cfg_mode == MODE_MIXED)
130 mode = rand() % 3;
131
132 sqe = io_uring_get_sqe(&ring);
133
134 if (mode == MODE_NONZC) {
135 io_uring_prep_send(sqe, fd, payload,
136 cfg_payload_len, msg_flags);
137 sqe->user_data = NONZC_TAG;
138 } else {
139 io_uring_prep_sendzc(sqe, fd, payload,
140 cfg_payload_len,
141 msg_flags, zc_flags);
142 if (mode == MODE_ZC_FIXED) {
143 sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
144 sqe->buf_index = buf_idx;
145 }
146 sqe->user_data = ZC_TAG;
147 }
148 }
149
150 ret = io_uring_submit(&ring);
151 if (ret != cfg_nr_reqs)
152 error(1, ret, "submit");
153
154 if (cfg_cork)
155 do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 0);
156 for (i = 0; i < cfg_nr_reqs; i++) {
157 ret = io_uring_wait_cqe(&ring, &cqe);
158 if (ret)
159 error(1, ret, "wait cqe");
160
161 if (cqe->user_data != NONZC_TAG &&
162 cqe->user_data != ZC_TAG)
163 error(1, -EINVAL, "invalid cqe->user_data");
164
165 if (cqe->flags & IORING_CQE_F_NOTIF) {
166 if (cqe->flags & IORING_CQE_F_MORE)
167 error(1, -EINVAL, "invalid notif flags");
168 if (compl_cqes <= 0)
169 error(1, -EINVAL, "notification mismatch");
170 compl_cqes--;
171 i--;
172 io_uring_cqe_seen(&ring);
173 continue;
174 }
175 if (cqe->flags & IORING_CQE_F_MORE) {
176 if (cqe->user_data != ZC_TAG)
177 error(1, cqe->res, "unexpected F_MORE");
178 compl_cqes++;
179 }
180 if (cqe->res >= 0) {
181 packets++;
182 bytes += cqe->res;
183 } else if (cqe->res != -EAGAIN) {
184 error(1, cqe->res, "send failed");
185 }
186 io_uring_cqe_seen(&ring);
187 }
188 } while (gettimeofday_ms() < tstop);
189
190 while (compl_cqes) {
191 ret = io_uring_wait_cqe(&ring, &cqe);
192 if (ret)
193 error(1, ret, "wait cqe");
194 if (cqe->flags & IORING_CQE_F_MORE)
195 error(1, -EINVAL, "invalid notif flags");
196 if (!(cqe->flags & IORING_CQE_F_NOTIF))
197 error(1, -EINVAL, "missing notif flag");
198
199 io_uring_cqe_seen(&ring);
200 compl_cqes--;
201 }
202
203 fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n",
204 packets, bytes >> 20,
205 packets / (cfg_runtime_ms / 1000),
206 (bytes >> 20) / (cfg_runtime_ms / 1000));
207
208 if (close(fd))
209 error(1, errno, "close");
210 }
211
212 static void do_test(int domain, int type, int protocol)
213 {
214 int i;
215
216 for (i = 0; i < IP_MAXPACKET; i++)
217 payload[i] = 'a' + (i % 26);
218 do_tx(domain, type, protocol);
219 }
220
221 static void usage(const char *filepath)
222 {
223 error(1, 0, "Usage: %s (-4|-6) (udp|tcp) -D<dst_ip> [-s<payload size>] "
224 "[-t<time s>] [-n<batch>] [-p<port>] [-m<mode>]", filepath);
225 }
226
227 static void parse_opts(int argc, char **argv)
228 {
229 const int max_payload_len = sizeof(payload) -
230 sizeof(struct ipv6hdr) -
231 sizeof(struct tcphdr) -
232 40 /* max tcp options */;
233 struct sockaddr_in6 *addr6 = (void *) &cfg_dst_addr;
234 struct sockaddr_in *addr4 = (void *) &cfg_dst_addr;
235 char *daddr = NULL;
236 int c;
237
238 if (argc <= 1)
239 usage(argv[0]);
240 cfg_payload_len = max_payload_len;
241
242 while ((c = getopt(argc, argv, "46D:p:s:t:n:c:m:")) != -1) {
243 switch (c) {
244 case '4':
245 if (cfg_family != PF_UNSPEC)
246 error(1, 0, "Pass one of -4 or -6");
247 cfg_family = PF_INET;
248 cfg_alen = sizeof(struct sockaddr_in);
249 break;
250 case '6':
251 if (cfg_family != PF_UNSPEC)
252 error(1, 0, "Pass one of -4 or -6");
253 cfg_family = PF_INET6;
254 cfg_alen = sizeof(struct sockaddr_in6);
255 break;
256 case 'D':
257 daddr = optarg;
258 break;
259 case 'p':
260 cfg_port = strtoul(optarg, NULL, 0);
261 break;
262 case 's':
263 cfg_payload_len = strtoul(optarg, NULL, 0);
264 break;
265 case 't':
266 cfg_runtime_ms = 200 + strtoul(optarg, NULL, 10) * 1000;
267 break;
268 case 'n':
269 cfg_nr_reqs = strtoul(optarg, NULL, 0);
270 break;
271 case 'c':
272 cfg_cork = strtol(optarg, NULL, 0);
273 break;
274 case 'm':
275 cfg_mode = strtol(optarg, NULL, 0);
276 break;
277 }
278 }
279
280 switch (cfg_family) {
281 case PF_INET:
282 memset(addr4, 0, sizeof(*addr4));
283 addr4->sin_family = AF_INET;
284 addr4->sin_port = htons(cfg_port);
285 if (daddr &&
286 inet_pton(AF_INET, daddr, &(addr4->sin_addr)) != 1)
287 error(1, 0, "ipv4 parse error: %s", daddr);
288 break;
289 case PF_INET6:
290 memset(addr6, 0, sizeof(*addr6));
291 addr6->sin6_family = AF_INET6;
292 addr6->sin6_port = htons(cfg_port);
293 if (daddr &&
294 inet_pton(AF_INET6, daddr, &(addr6->sin6_addr)) != 1)
295 error(1, 0, "ipv6 parse error: %s", daddr);
296 break;
297 default:
298 error(1, 0, "illegal domain");
299 }
300
301 if (cfg_payload_len > max_payload_len)
302 error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
303 if (optind != argc - 1)
304 usage(argv[0]);
305 }
306
307 int main(int argc, char **argv)
308 {
309 const char *cfg_test = argv[argc - 1];
310
311 parse_opts(argc, argv);
312
313 if (!strcmp(cfg_test, "tcp"))
314 do_test(cfg_family, SOCK_STREAM, 0);
315 else if (!strcmp(cfg_test, "udp"))
316 do_test(cfg_family, SOCK_DGRAM, 0);
317 else
318 error(1, 0, "unknown cfg_test %s", cfg_test);
319 return 0;
320 }
321
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.