1 #ifndef BLK_THROTTLE_H 1
2 #define BLK_THROTTLE_H
3
4 #include "blk-cgroup-rwstat.h"
5
6 /*
7 * To implement hierarchical throttling, throt
8 * are dispatched upwards level by level until
9 * issued. When dispatching bios from the chi
10 * level, if the bios are dispatched into a si
11 * of a local or child group which can queue m
12 * the list starving others.
13 *
14 * To avoid such starvation, dispatched bios a
15 * according to where they came from. When th
16 * the parent, they're popped in round-robin o
17 * hogs the dispatch window.
18 *
19 * throtl_qnode is used to keep the queued bio
20 * Bios are queued to throtl_qnode which in tu
21 * throtl_service_queue and then dispatched in
22 *
23 * It's also used to track the reference count
24 * belongs to a throtl_grp and gets queued on
25 * incrementing the reference of the associate
26 * queued and decrementing when dequeued is en
27 * tree pinned while bios are in flight.
28 */
29 struct throtl_qnode {
30 struct list_head node;
31 struct bio_list bios;
32 struct throtl_grp *tg;
33 };
34
35 struct throtl_service_queue {
36 struct throtl_service_queue *parent_sq
37
38 /*
39 * Bios queued directly to this servic
40 * children throtl_grp's.
41 */
42 struct list_head queued[2];
43 unsigned int nr_queued[2];
44
45 /*
46 * RB tree of active children throtl_g
47 * their ->disptime.
48 */
49 struct rb_root_cached pending_tree;
50 unsigned int nr_pending;
51 unsigned long first_pending_
52 struct timer_list pending_timer;
53 };
54
55 enum tg_state_flags {
56 THROTL_TG_PENDING = 1 << 0,
57 THROTL_TG_WAS_EMPTY = 1 << 1,
58 THROTL_TG_CANCELING = 1 << 2,
59 };
60
61 struct throtl_grp {
62 /* must be the first member */
63 struct blkg_policy_data pd;
64
65 /* active throtl group service_queue m
66 struct rb_node rb_node;
67
68 /* throtl_data this group belongs to *
69 struct throtl_data *td;
70
71 /* this group's service queue */
72 struct throtl_service_queue service_qu
73
74 /*
75 * qnode_on_self is used when bios are
76 * throtl_grp so that local bios compe
77 * dispatched from children. qnode_on
78 * dispatched from this throtl_grp int
79 * with the sibling qnode_on_parents a
80 * qnode_on_self.
81 */
82 struct throtl_qnode qnode_on_self[2];
83 struct throtl_qnode qnode_on_parent[2]
84
85 /*
86 * Dispatch time in jiffies. This is t
87 * will unthrottle and is ready to dis
88 * key to sort active groups in servic
89 */
90 unsigned long disptime;
91
92 unsigned int flags;
93
94 /* are there any throtl rules between
95 bool has_rules_bps[2];
96 bool has_rules_iops[2];
97
98 /* bytes per second rate limits */
99 uint64_t bps[2];
100
101 /* IOPS limits */
102 unsigned int iops[2];
103
104 /* Number of bytes dispatched in curre
105 uint64_t bytes_disp[2];
106 /* Number of bio's dispatched in curre
107 unsigned int io_disp[2];
108
109 unsigned long last_low_overflow_time[2
110
111 uint64_t last_bytes_disp[2];
112 unsigned int last_io_disp[2];
113
114 /*
115 * The following two fields are update
116 * submitted while some bios are still
117 * bytes/ios are waited already in pre
118 * be used to calculate wait time unde
119 */
120 long long carryover_bytes[2];
121 int carryover_ios[2];
122
123 unsigned long last_check_time;
124
125 /* When did we start a new slice */
126 unsigned long slice_start[2];
127 unsigned long slice_end[2];
128
129 struct blkg_rwstat stat_bytes;
130 struct blkg_rwstat stat_ios;
131 };
132
133 extern struct blkcg_policy blkcg_policy_throtl
134
135 static inline struct throtl_grp *pd_to_tg(stru
136 {
137 return pd ? container_of(pd, struct th
138 }
139
140 static inline struct throtl_grp *blkg_to_tg(st
141 {
142 return pd_to_tg(blkg_to_pd(blkg, &blkc
143 }
144
145 /*
146 * Internal throttling interface
147 */
148 #ifndef CONFIG_BLK_DEV_THROTTLING
149 static inline void blk_throtl_exit(struct gend
150 static inline bool blk_throtl_bio(struct bio *
151 static inline void blk_throtl_cancel_bios(stru
152 #else /* CONFIG_BLK_DEV_THROTTLING */
153 void blk_throtl_exit(struct gendisk *disk);
154 bool __blk_throtl_bio(struct bio *bio);
155 void blk_throtl_cancel_bios(struct gendisk *di
156
157 static inline bool blk_throtl_activated(struct
158 {
159 return q->td != NULL;
160 }
161
162 static inline bool blk_should_throtl(struct bi
163 {
164 struct throtl_grp *tg;
165 int rw = bio_data_dir(bio);
166
167 /*
168 * This is called under bio_queue_ente
169 * the activation of blk-throtl, which
170 * blk_mq_freeze_queue().
171 */
172 if (!blk_throtl_activated(bio->bi_bdev
173 return false;
174
175 tg = blkg_to_tg(bio->bi_blkg);
176 if (!cgroup_subsys_on_dfl(io_cgrp_subs
177 if (!bio_flagged(bio, BIO_CGRO
178 bio_set_flag(bio, BIO_
179 blkg_rwstat_add(&tg->s
180 bio->b
181 }
182 blkg_rwstat_add(&tg->stat_ios,
183 }
184
185 /* iops limit is always counted */
186 if (tg->has_rules_iops[rw])
187 return true;
188
189 if (tg->has_rules_bps[rw] && !bio_flag
190 return true;
191
192 return false;
193 }
194
195 static inline bool blk_throtl_bio(struct bio *
196 {
197
198 if (!blk_should_throtl(bio))
199 return false;
200
201 return __blk_throtl_bio(bio);
202 }
203 #endif /* CONFIG_BLK_DEV_THROTTLING */
204
205 #endif
206
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.