1 /* SPDX-License-Identifier: GPL-2.0 */ 1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __NET_TC_WRAPPER_H 2 #ifndef __NET_TC_WRAPPER_H
3 #define __NET_TC_WRAPPER_H 3 #define __NET_TC_WRAPPER_H
4 4
5 #include <net/pkt_cls.h> 5 #include <net/pkt_cls.h>
6 6
7 #if IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) 7 #if IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)
8 8
9 #include <linux/cpufeature.h> 9 #include <linux/cpufeature.h>
10 #include <linux/static_key.h> 10 #include <linux/static_key.h>
11 #include <linux/indirect_call_wrapper.h> 11 #include <linux/indirect_call_wrapper.h>
12 12
13 #define TC_INDIRECT_SCOPE 13 #define TC_INDIRECT_SCOPE
14 14
15 extern struct static_key_false tc_skip_wrapper 15 extern struct static_key_false tc_skip_wrapper;
16 16
17 /* TC Actions */ 17 /* TC Actions */
18 #ifdef CONFIG_NET_CLS_ACT 18 #ifdef CONFIG_NET_CLS_ACT
19 19
20 #define TC_INDIRECT_ACTION_DECLARE(fname) 20 #define TC_INDIRECT_ACTION_DECLARE(fname) \
21 INDIRECT_CALLABLE_DECLARE(int fname(st 21 INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
22 co 22 const struct tc_action *a, \
23 st 23 struct tcf_result *res))
24 24
25 TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act); 25 TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act);
26 TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act); 26 TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act);
27 TC_INDIRECT_ACTION_DECLARE(tcf_csum_act); 27 TC_INDIRECT_ACTION_DECLARE(tcf_csum_act);
28 TC_INDIRECT_ACTION_DECLARE(tcf_ct_act); 28 TC_INDIRECT_ACTION_DECLARE(tcf_ct_act);
29 TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act); 29 TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act);
30 TC_INDIRECT_ACTION_DECLARE(tcf_gact_act); 30 TC_INDIRECT_ACTION_DECLARE(tcf_gact_act);
31 TC_INDIRECT_ACTION_DECLARE(tcf_gate_act); 31 TC_INDIRECT_ACTION_DECLARE(tcf_gate_act);
32 TC_INDIRECT_ACTION_DECLARE(tcf_ife_act); 32 TC_INDIRECT_ACTION_DECLARE(tcf_ife_act);
33 TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act); 33 TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act);
34 TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act); 34 TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act);
35 TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act); 35 TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act);
36 TC_INDIRECT_ACTION_DECLARE(tcf_nat_act); 36 TC_INDIRECT_ACTION_DECLARE(tcf_nat_act);
37 TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act); 37 TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act);
38 TC_INDIRECT_ACTION_DECLARE(tcf_police_act); 38 TC_INDIRECT_ACTION_DECLARE(tcf_police_act);
39 TC_INDIRECT_ACTION_DECLARE(tcf_sample_act); 39 TC_INDIRECT_ACTION_DECLARE(tcf_sample_act);
40 TC_INDIRECT_ACTION_DECLARE(tcf_simp_act); 40 TC_INDIRECT_ACTION_DECLARE(tcf_simp_act);
41 TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act); 41 TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act);
42 TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act); 42 TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act);
43 TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act); 43 TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act);
44 TC_INDIRECT_ACTION_DECLARE(tunnel_key_act); 44 TC_INDIRECT_ACTION_DECLARE(tunnel_key_act);
45 45
46 static inline int tc_act(struct sk_buff *skb, 46 static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
47 struct tcf_result * 47 struct tcf_result *res)
48 { 48 {
49 if (static_branch_likely(&tc_skip_wrap 49 if (static_branch_likely(&tc_skip_wrapper))
50 goto skip; 50 goto skip;
51 51
52 #if IS_BUILTIN(CONFIG_NET_ACT_GACT) 52 #if IS_BUILTIN(CONFIG_NET_ACT_GACT)
53 if (a->ops->act == tcf_gact_act) 53 if (a->ops->act == tcf_gact_act)
54 return tcf_gact_act(skb, a, re 54 return tcf_gact_act(skb, a, res);
55 #endif 55 #endif
56 #if IS_BUILTIN(CONFIG_NET_ACT_MIRRED) 56 #if IS_BUILTIN(CONFIG_NET_ACT_MIRRED)
57 if (a->ops->act == tcf_mirred_act) 57 if (a->ops->act == tcf_mirred_act)
58 return tcf_mirred_act(skb, a, 58 return tcf_mirred_act(skb, a, res);
59 #endif 59 #endif
60 #if IS_BUILTIN(CONFIG_NET_ACT_PEDIT) 60 #if IS_BUILTIN(CONFIG_NET_ACT_PEDIT)
61 if (a->ops->act == tcf_pedit_act) 61 if (a->ops->act == tcf_pedit_act)
62 return tcf_pedit_act(skb, a, r 62 return tcf_pedit_act(skb, a, res);
63 #endif 63 #endif
64 #if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT) 64 #if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT)
65 if (a->ops->act == tcf_skbedit_act) 65 if (a->ops->act == tcf_skbedit_act)
66 return tcf_skbedit_act(skb, a, 66 return tcf_skbedit_act(skb, a, res);
67 #endif 67 #endif
68 #if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD) 68 #if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD)
69 if (a->ops->act == tcf_skbmod_act) 69 if (a->ops->act == tcf_skbmod_act)
70 return tcf_skbmod_act(skb, a, 70 return tcf_skbmod_act(skb, a, res);
71 #endif 71 #endif
72 #if IS_BUILTIN(CONFIG_NET_ACT_POLICE) 72 #if IS_BUILTIN(CONFIG_NET_ACT_POLICE)
73 if (a->ops->act == tcf_police_act) 73 if (a->ops->act == tcf_police_act)
74 return tcf_police_act(skb, a, 74 return tcf_police_act(skb, a, res);
75 #endif 75 #endif
76 #if IS_BUILTIN(CONFIG_NET_ACT_BPF) 76 #if IS_BUILTIN(CONFIG_NET_ACT_BPF)
77 if (a->ops->act == tcf_bpf_act) 77 if (a->ops->act == tcf_bpf_act)
78 return tcf_bpf_act(skb, a, res 78 return tcf_bpf_act(skb, a, res);
79 #endif 79 #endif
80 #if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK) 80 #if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK)
81 if (a->ops->act == tcf_connmark_act) 81 if (a->ops->act == tcf_connmark_act)
82 return tcf_connmark_act(skb, a 82 return tcf_connmark_act(skb, a, res);
83 #endif 83 #endif
84 #if IS_BUILTIN(CONFIG_NET_ACT_CSUM) 84 #if IS_BUILTIN(CONFIG_NET_ACT_CSUM)
85 if (a->ops->act == tcf_csum_act) 85 if (a->ops->act == tcf_csum_act)
86 return tcf_csum_act(skb, a, re 86 return tcf_csum_act(skb, a, res);
87 #endif 87 #endif
88 #if IS_BUILTIN(CONFIG_NET_ACT_CT) 88 #if IS_BUILTIN(CONFIG_NET_ACT_CT)
89 if (a->ops->act == tcf_ct_act) 89 if (a->ops->act == tcf_ct_act)
90 return tcf_ct_act(skb, a, res) 90 return tcf_ct_act(skb, a, res);
91 #endif 91 #endif
92 #if IS_BUILTIN(CONFIG_NET_ACT_CTINFO) 92 #if IS_BUILTIN(CONFIG_NET_ACT_CTINFO)
93 if (a->ops->act == tcf_ctinfo_act) 93 if (a->ops->act == tcf_ctinfo_act)
94 return tcf_ctinfo_act(skb, a, 94 return tcf_ctinfo_act(skb, a, res);
95 #endif 95 #endif
96 #if IS_BUILTIN(CONFIG_NET_ACT_GATE) 96 #if IS_BUILTIN(CONFIG_NET_ACT_GATE)
97 if (a->ops->act == tcf_gate_act) 97 if (a->ops->act == tcf_gate_act)
98 return tcf_gate_act(skb, a, re 98 return tcf_gate_act(skb, a, res);
99 #endif 99 #endif
100 #if IS_BUILTIN(CONFIG_NET_ACT_MPLS) 100 #if IS_BUILTIN(CONFIG_NET_ACT_MPLS)
101 if (a->ops->act == tcf_mpls_act) 101 if (a->ops->act == tcf_mpls_act)
102 return tcf_mpls_act(skb, a, re 102 return tcf_mpls_act(skb, a, res);
103 #endif 103 #endif
104 #if IS_BUILTIN(CONFIG_NET_ACT_NAT) 104 #if IS_BUILTIN(CONFIG_NET_ACT_NAT)
105 if (a->ops->act == tcf_nat_act) 105 if (a->ops->act == tcf_nat_act)
106 return tcf_nat_act(skb, a, res 106 return tcf_nat_act(skb, a, res);
107 #endif 107 #endif
108 #if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY) 108 #if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY)
109 if (a->ops->act == tunnel_key_act) 109 if (a->ops->act == tunnel_key_act)
110 return tunnel_key_act(skb, a, 110 return tunnel_key_act(skb, a, res);
111 #endif 111 #endif
112 #if IS_BUILTIN(CONFIG_NET_ACT_VLAN) 112 #if IS_BUILTIN(CONFIG_NET_ACT_VLAN)
113 if (a->ops->act == tcf_vlan_act) 113 if (a->ops->act == tcf_vlan_act)
114 return tcf_vlan_act(skb, a, re 114 return tcf_vlan_act(skb, a, res);
115 #endif 115 #endif
116 #if IS_BUILTIN(CONFIG_NET_ACT_IFE) 116 #if IS_BUILTIN(CONFIG_NET_ACT_IFE)
117 if (a->ops->act == tcf_ife_act) 117 if (a->ops->act == tcf_ife_act)
118 return tcf_ife_act(skb, a, res 118 return tcf_ife_act(skb, a, res);
119 #endif 119 #endif
120 #if IS_BUILTIN(CONFIG_NET_ACT_SIMP) 120 #if IS_BUILTIN(CONFIG_NET_ACT_SIMP)
121 if (a->ops->act == tcf_simp_act) 121 if (a->ops->act == tcf_simp_act)
122 return tcf_simp_act(skb, a, re 122 return tcf_simp_act(skb, a, res);
123 #endif 123 #endif
124 #if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE) 124 #if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE)
125 if (a->ops->act == tcf_sample_act) 125 if (a->ops->act == tcf_sample_act)
126 return tcf_sample_act(skb, a, 126 return tcf_sample_act(skb, a, res);
127 #endif 127 #endif
128 128
129 skip: 129 skip:
130 return a->ops->act(skb, a, res); 130 return a->ops->act(skb, a, res);
131 } 131 }
132 132
133 #endif /* CONFIG_NET_CLS_ACT */ 133 #endif /* CONFIG_NET_CLS_ACT */
134 134
135 /* TC Filters */ 135 /* TC Filters */
136 #ifdef CONFIG_NET_CLS 136 #ifdef CONFIG_NET_CLS
137 137
138 #define TC_INDIRECT_FILTER_DECLARE(fname) 138 #define TC_INDIRECT_FILTER_DECLARE(fname) \
139 INDIRECT_CALLABLE_DECLARE(int fname(st 139 INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
140 co 140 const struct tcf_proto *tp, \
141 st 141 struct tcf_result *res))
142 142
143 TC_INDIRECT_FILTER_DECLARE(basic_classify); 143 TC_INDIRECT_FILTER_DECLARE(basic_classify);
144 TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify); 144 TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify);
145 TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify 145 TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify);
146 TC_INDIRECT_FILTER_DECLARE(fl_classify); 146 TC_INDIRECT_FILTER_DECLARE(fl_classify);
147 TC_INDIRECT_FILTER_DECLARE(flow_classify); 147 TC_INDIRECT_FILTER_DECLARE(flow_classify);
148 TC_INDIRECT_FILTER_DECLARE(fw_classify); 148 TC_INDIRECT_FILTER_DECLARE(fw_classify);
149 TC_INDIRECT_FILTER_DECLARE(mall_classify); 149 TC_INDIRECT_FILTER_DECLARE(mall_classify);
150 TC_INDIRECT_FILTER_DECLARE(route4_classify); 150 TC_INDIRECT_FILTER_DECLARE(route4_classify);
151 TC_INDIRECT_FILTER_DECLARE(u32_classify); 151 TC_INDIRECT_FILTER_DECLARE(u32_classify);
152 152
153 static inline int tc_classify(struct sk_buff * 153 static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
154 struct tcf_res 154 struct tcf_result *res)
155 { 155 {
156 if (static_branch_likely(&tc_skip_wrap 156 if (static_branch_likely(&tc_skip_wrapper))
157 goto skip; 157 goto skip;
158 158
159 #if IS_BUILTIN(CONFIG_NET_CLS_BPF) 159 #if IS_BUILTIN(CONFIG_NET_CLS_BPF)
160 if (tp->classify == cls_bpf_classify) 160 if (tp->classify == cls_bpf_classify)
161 return cls_bpf_classify(skb, t 161 return cls_bpf_classify(skb, tp, res);
162 #endif 162 #endif
163 #if IS_BUILTIN(CONFIG_NET_CLS_U32) 163 #if IS_BUILTIN(CONFIG_NET_CLS_U32)
164 if (tp->classify == u32_classify) 164 if (tp->classify == u32_classify)
165 return u32_classify(skb, tp, r 165 return u32_classify(skb, tp, res);
166 #endif 166 #endif
167 #if IS_BUILTIN(CONFIG_NET_CLS_FLOWER) 167 #if IS_BUILTIN(CONFIG_NET_CLS_FLOWER)
168 if (tp->classify == fl_classify) 168 if (tp->classify == fl_classify)
169 return fl_classify(skb, tp, re 169 return fl_classify(skb, tp, res);
170 #endif 170 #endif
171 #if IS_BUILTIN(CONFIG_NET_CLS_FW) 171 #if IS_BUILTIN(CONFIG_NET_CLS_FW)
172 if (tp->classify == fw_classify) 172 if (tp->classify == fw_classify)
173 return fw_classify(skb, tp, re 173 return fw_classify(skb, tp, res);
174 #endif 174 #endif
175 #if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL) 175 #if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL)
176 if (tp->classify == mall_classify) 176 if (tp->classify == mall_classify)
177 return mall_classify(skb, tp, 177 return mall_classify(skb, tp, res);
178 #endif 178 #endif
179 #if IS_BUILTIN(CONFIG_NET_CLS_BASIC) 179 #if IS_BUILTIN(CONFIG_NET_CLS_BASIC)
180 if (tp->classify == basic_classify) 180 if (tp->classify == basic_classify)
181 return basic_classify(skb, tp, 181 return basic_classify(skb, tp, res);
182 #endif 182 #endif
183 #if IS_BUILTIN(CONFIG_NET_CLS_CGROUP) 183 #if IS_BUILTIN(CONFIG_NET_CLS_CGROUP)
184 if (tp->classify == cls_cgroup_classif 184 if (tp->classify == cls_cgroup_classify)
185 return cls_cgroup_classify(skb 185 return cls_cgroup_classify(skb, tp, res);
186 #endif 186 #endif
187 #if IS_BUILTIN(CONFIG_NET_CLS_FLOW) 187 #if IS_BUILTIN(CONFIG_NET_CLS_FLOW)
188 if (tp->classify == flow_classify) 188 if (tp->classify == flow_classify)
189 return flow_classify(skb, tp, 189 return flow_classify(skb, tp, res);
190 #endif 190 #endif
191 #if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4) 191 #if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4)
192 if (tp->classify == route4_classify) 192 if (tp->classify == route4_classify)
193 return route4_classify(skb, tp 193 return route4_classify(skb, tp, res);
194 #endif 194 #endif
195 195
196 skip: 196 skip:
197 return tp->classify(skb, tp, res); 197 return tp->classify(skb, tp, res);
198 } 198 }
199 199
200 #endif /* CONFIG_NET_CLS */ 200 #endif /* CONFIG_NET_CLS */
201 201
202 static inline void tc_wrapper_init(void) 202 static inline void tc_wrapper_init(void)
203 { 203 {
204 #ifdef CONFIG_X86 204 #ifdef CONFIG_X86
205 if (!cpu_feature_enabled(X86_FEATURE_R 205 if (!cpu_feature_enabled(X86_FEATURE_RETPOLINE))
206 static_branch_enable(&tc_skip_ 206 static_branch_enable(&tc_skip_wrapper);
207 #endif 207 #endif
208 } 208 }
209 209
210 #else 210 #else
211 211
212 #define TC_INDIRECT_SCOPE static 212 #define TC_INDIRECT_SCOPE static
213 213
214 static inline int tc_act(struct sk_buff *skb, 214 static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
215 struct tcf_result * 215 struct tcf_result *res)
216 { 216 {
217 return a->ops->act(skb, a, res); 217 return a->ops->act(skb, a, res);
218 } 218 }
219 219
220 static inline int tc_classify(struct sk_buff * 220 static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
221 struct tcf_res 221 struct tcf_result *res)
222 { 222 {
223 return tp->classify(skb, tp, res); 223 return tp->classify(skb, tp, res);
224 } 224 }
225 225
226 static inline void tc_wrapper_init(void) 226 static inline void tc_wrapper_init(void)
227 { 227 {
228 } 228 }
229 229
230 #endif 230 #endif
231 231
232 #endif /* __NET_TC_WRAPPER_H */ 232 #endif /* __NET_TC_WRAPPER_H */
233 233
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.