1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Kernel Connection Multiplexor
4 *
5 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com>
6 */
7
8 #ifndef __NET_KCM_H_
9 #define __NET_KCM_H_
10
11 #include <linux/skbuff.h>
12 #include <net/sock.h>
13 #include <net/strparser.h>
14 #include <uapi/linux/kcm.h>
15
16 extern unsigned int kcm_net_id;
17
18 #define KCM_STATS_ADD(stat, count) ((stat) += (count))
19 #define KCM_STATS_INCR(stat) ((stat)++)
20
21 struct kcm_psock_stats {
22 unsigned long long tx_msgs;
23 unsigned long long tx_bytes;
24 unsigned long long reserved;
25 unsigned long long unreserved;
26 unsigned int tx_aborts;
27 };
28
29 struct kcm_mux_stats {
30 unsigned long long rx_msgs;
31 unsigned long long rx_bytes;
32 unsigned long long tx_msgs;
33 unsigned long long tx_bytes;
34 unsigned int rx_ready_drops;
35 unsigned int tx_retries;
36 unsigned int psock_attach;
37 unsigned int psock_unattach_rsvd;
38 unsigned int psock_unattach;
39 };
40
41 struct kcm_stats {
42 unsigned long long rx_msgs;
43 unsigned long long rx_bytes;
44 unsigned long long tx_msgs;
45 unsigned long long tx_bytes;
46 };
47
48 struct kcm_tx_msg {
49 unsigned int sent;
50 unsigned int frag_offset;
51 unsigned int msg_flags;
52 bool started_tx;
53 struct sk_buff *frag_skb;
54 struct sk_buff *last_skb;
55 };
56
57 /* Socket structure for KCM client sockets */
58 struct kcm_sock {
59 struct sock sk;
60 struct kcm_mux *mux;
61 struct list_head kcm_sock_list;
62 int index;
63 u32 done : 1;
64 struct work_struct done_work;
65
66 struct kcm_stats stats;
67
68 /* Transmit */
69 struct kcm_psock *tx_psock;
70 struct work_struct tx_work;
71 struct list_head wait_psock_list;
72 struct sk_buff *seq_skb;
73 u32 tx_stopped : 1;
74
75 /* Don't use bit fields here, these are set under different locks */
76 bool tx_wait;
77 bool tx_wait_more;
78
79 /* Receive */
80 struct kcm_psock *rx_psock;
81 struct list_head wait_rx_list; /* KCMs waiting for receiving */
82 bool rx_wait;
83 u32 rx_disabled : 1;
84 };
85
86 struct bpf_prog;
87
88 /* Structure for an attached lower socket */
89 struct kcm_psock {
90 struct sock *sk;
91 struct strparser strp;
92 struct kcm_mux *mux;
93 int index;
94
95 u32 tx_stopped : 1;
96 u32 done : 1;
97 u32 unattaching : 1;
98
99 void (*save_state_change)(struct sock *sk);
100 void (*save_data_ready)(struct sock *sk);
101 void (*save_write_space)(struct sock *sk);
102
103 struct list_head psock_list;
104
105 struct kcm_psock_stats stats;
106
107 /* Receive */
108 struct list_head psock_ready_list;
109 struct bpf_prog *bpf_prog;
110 struct kcm_sock *rx_kcm;
111 unsigned long long saved_rx_bytes;
112 unsigned long long saved_rx_msgs;
113 struct sk_buff *ready_rx_msg;
114
115 /* Transmit */
116 struct kcm_sock *tx_kcm;
117 struct list_head psock_avail_list;
118 unsigned long long saved_tx_bytes;
119 unsigned long long saved_tx_msgs;
120 };
121
122 /* Per net MUX list */
123 struct kcm_net {
124 struct mutex mutex;
125 struct kcm_psock_stats aggregate_psock_stats;
126 struct kcm_mux_stats aggregate_mux_stats;
127 struct strp_aggr_stats aggregate_strp_stats;
128 struct list_head mux_list;
129 int count;
130 };
131
132 /* Structure for a MUX */
133 struct kcm_mux {
134 struct list_head kcm_mux_list;
135 struct rcu_head rcu;
136 struct kcm_net *knet;
137
138 struct list_head kcm_socks; /* All KCM sockets on MUX */
139 int kcm_socks_cnt; /* Total KCM socket count for MUX */
140 struct list_head psocks; /* List of all psocks on MUX */
141 int psocks_cnt; /* Total attached sockets */
142
143 struct kcm_mux_stats stats;
144 struct kcm_psock_stats aggregate_psock_stats;
145 struct strp_aggr_stats aggregate_strp_stats;
146
147 /* Receive */
148 spinlock_t rx_lock ____cacheline_aligned_in_smp;
149 struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */
150 struct list_head psocks_ready; /* List of psocks with a msg ready */
151 struct sk_buff_head rx_hold_queue;
152
153 /* Transmit */
154 spinlock_t lock ____cacheline_aligned_in_smp; /* TX and mux locking */
155 struct list_head psocks_avail; /* List of available psocks */
156 struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */
157 };
158
159 #ifdef CONFIG_PROC_FS
160 int kcm_proc_init(void);
161 void kcm_proc_exit(void);
162 #else
163 static inline int kcm_proc_init(void) { return 0; }
164 static inline void kcm_proc_exit(void) { }
165 #endif
166
167 static inline void aggregate_psock_stats(struct kcm_psock_stats *stats,
168 struct kcm_psock_stats *agg_stats)
169 {
170 /* Save psock statistics in the mux when psock is being unattached. */
171
172 #define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat)
173 SAVE_PSOCK_STATS(tx_msgs);
174 SAVE_PSOCK_STATS(tx_bytes);
175 SAVE_PSOCK_STATS(reserved);
176 SAVE_PSOCK_STATS(unreserved);
177 SAVE_PSOCK_STATS(tx_aborts);
178 #undef SAVE_PSOCK_STATS
179 }
180
181 static inline void aggregate_mux_stats(struct kcm_mux_stats *stats,
182 struct kcm_mux_stats *agg_stats)
183 {
184 /* Save psock statistics in the mux when psock is being unattached. */
185
186 #define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat)
187 SAVE_MUX_STATS(rx_msgs);
188 SAVE_MUX_STATS(rx_bytes);
189 SAVE_MUX_STATS(tx_msgs);
190 SAVE_MUX_STATS(tx_bytes);
191 SAVE_MUX_STATS(rx_ready_drops);
192 SAVE_MUX_STATS(psock_attach);
193 SAVE_MUX_STATS(psock_unattach_rsvd);
194 SAVE_MUX_STATS(psock_unattach);
195 #undef SAVE_MUX_STATS
196 }
197
198 #endif /* __NET_KCM_H_ */
199
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.