1 #ifndef BLK_THROTTLE_H 1 #ifndef BLK_THROTTLE_H
2 #define BLK_THROTTLE_H 2 #define BLK_THROTTLE_H
3 3
4 #include "blk-cgroup-rwstat.h" 4 #include "blk-cgroup-rwstat.h"
5 5
6 /* 6 /*
7 * To implement hierarchical throttling, throt 7 * To implement hierarchical throttling, throtl_grps form a tree and bios
8 * are dispatched upwards level by level until 8 * are dispatched upwards level by level until they reach the top and get
9 * issued. When dispatching bios from the chi 9 * issued. When dispatching bios from the children and local group at each
10 * level, if the bios are dispatched into a si 10 * level, if the bios are dispatched into a single bio_list, there's a risk
11 * of a local or child group which can queue m 11 * of a local or child group which can queue many bios at once filling up
12 * the list starving others. 12 * the list starving others.
13 * 13 *
14 * To avoid such starvation, dispatched bios a 14 * To avoid such starvation, dispatched bios are queued separately
15 * according to where they came from. When th 15 * according to where they came from. When they are again dispatched to
16 * the parent, they're popped in round-robin o 16 * the parent, they're popped in round-robin order so that no single source
17 * hogs the dispatch window. 17 * hogs the dispatch window.
18 * 18 *
19 * throtl_qnode is used to keep the queued bio 19 * throtl_qnode is used to keep the queued bios separated by their sources.
20 * Bios are queued to throtl_qnode which in tu 20 * Bios are queued to throtl_qnode which in turn is queued to
21 * throtl_service_queue and then dispatched in 21 * throtl_service_queue and then dispatched in round-robin order.
22 * 22 *
23 * It's also used to track the reference count 23 * It's also used to track the reference counts on blkg's. A qnode always
24 * belongs to a throtl_grp and gets queued on 24 * belongs to a throtl_grp and gets queued on itself or the parent, so
25 * incrementing the reference of the associate 25 * incrementing the reference of the associated throtl_grp when a qnode is
26 * queued and decrementing when dequeued is en 26 * queued and decrementing when dequeued is enough to keep the whole blkg
27 * tree pinned while bios are in flight. 27 * tree pinned while bios are in flight.
28 */ 28 */
29 struct throtl_qnode { 29 struct throtl_qnode {
30 struct list_head node; 30 struct list_head node; /* service_queue->queued[] */
31 struct bio_list bios; 31 struct bio_list bios; /* queued bios */
32 struct throtl_grp *tg; 32 struct throtl_grp *tg; /* tg this qnode belongs to */
33 }; 33 };
34 34
35 struct throtl_service_queue { 35 struct throtl_service_queue {
36 struct throtl_service_queue *parent_sq 36 struct throtl_service_queue *parent_sq; /* the parent service_queue */
37 37
38 /* 38 /*
39 * Bios queued directly to this servic 39 * Bios queued directly to this service_queue or dispatched from
40 * children throtl_grp's. 40 * children throtl_grp's.
41 */ 41 */
42 struct list_head queued[2]; 42 struct list_head queued[2]; /* throtl_qnode [READ/WRITE] */
43 unsigned int nr_queued[2]; 43 unsigned int nr_queued[2]; /* number of queued bios */
44 44
45 /* 45 /*
46 * RB tree of active children throtl_g 46 * RB tree of active children throtl_grp's, which are sorted by
47 * their ->disptime. 47 * their ->disptime.
48 */ 48 */
49 struct rb_root_cached pending_tree; 49 struct rb_root_cached pending_tree; /* RB tree of active tgs */
50 unsigned int nr_pending; 50 unsigned int nr_pending; /* # queued in the tree */
51 unsigned long first_pending_ 51 unsigned long first_pending_disptime; /* disptime of the first tg */
52 struct timer_list pending_timer; 52 struct timer_list pending_timer; /* fires on first_pending_disptime */
53 }; 53 };
54 54
55 enum tg_state_flags { 55 enum tg_state_flags {
56 THROTL_TG_PENDING = 1 << 0, 56 THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */
57 THROTL_TG_WAS_EMPTY = 1 << 1, 57 THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */
58 THROTL_TG_CANCELING = 1 << 2, !! 58 THROTL_TG_HAS_IOPS_LIMIT = 1 << 2, /* tg has iops limit */
>> 59 THROTL_TG_CANCELING = 1 << 3, /* starts to cancel bio */
>> 60 };
>> 61
>> 62 enum {
>> 63 LIMIT_LOW,
>> 64 LIMIT_MAX,
>> 65 LIMIT_CNT,
59 }; 66 };
60 67
61 struct throtl_grp { 68 struct throtl_grp {
62 /* must be the first member */ 69 /* must be the first member */
63 struct blkg_policy_data pd; 70 struct blkg_policy_data pd;
64 71
65 /* active throtl group service_queue m 72 /* active throtl group service_queue member */
66 struct rb_node rb_node; 73 struct rb_node rb_node;
67 74
68 /* throtl_data this group belongs to * 75 /* throtl_data this group belongs to */
69 struct throtl_data *td; 76 struct throtl_data *td;
70 77
71 /* this group's service queue */ 78 /* this group's service queue */
72 struct throtl_service_queue service_qu 79 struct throtl_service_queue service_queue;
73 80
74 /* 81 /*
75 * qnode_on_self is used when bios are 82 * qnode_on_self is used when bios are directly queued to this
76 * throtl_grp so that local bios compe 83 * throtl_grp so that local bios compete fairly with bios
77 * dispatched from children. qnode_on 84 * dispatched from children. qnode_on_parent is used when bios are
78 * dispatched from this throtl_grp int 85 * dispatched from this throtl_grp into its parent and will compete
79 * with the sibling qnode_on_parents a 86 * with the sibling qnode_on_parents and the parent's
80 * qnode_on_self. 87 * qnode_on_self.
81 */ 88 */
82 struct throtl_qnode qnode_on_self[2]; 89 struct throtl_qnode qnode_on_self[2];
83 struct throtl_qnode qnode_on_parent[2] 90 struct throtl_qnode qnode_on_parent[2];
84 91
85 /* 92 /*
86 * Dispatch time in jiffies. This is t 93 * Dispatch time in jiffies. This is the estimated time when group
87 * will unthrottle and is ready to dis 94 * will unthrottle and is ready to dispatch more bio. It is used as
88 * key to sort active groups in servic 95 * key to sort active groups in service tree.
89 */ 96 */
90 unsigned long disptime; 97 unsigned long disptime;
91 98
92 unsigned int flags; 99 unsigned int flags;
93 100
94 /* are there any throtl rules between 101 /* are there any throtl rules between this group and td? */
95 bool has_rules_bps[2]; !! 102 bool has_rules[2];
96 bool has_rules_iops[2]; <<
97 <<
98 /* bytes per second rate limits */ <<
99 uint64_t bps[2]; <<
100 103
101 /* IOPS limits */ !! 104 /* internally used bytes per second rate limits */
102 unsigned int iops[2]; !! 105 uint64_t bps[2][LIMIT_CNT];
>> 106 /* user configured bps limits */
>> 107 uint64_t bps_conf[2][LIMIT_CNT];
>> 108
>> 109 /* internally used IOPS limits */
>> 110 unsigned int iops[2][LIMIT_CNT];
>> 111 /* user configured IOPS limits */
>> 112 unsigned int iops_conf[2][LIMIT_CNT];
103 113
104 /* Number of bytes dispatched in curre 114 /* Number of bytes dispatched in current slice */
105 uint64_t bytes_disp[2]; 115 uint64_t bytes_disp[2];
106 /* Number of bio's dispatched in curre 116 /* Number of bio's dispatched in current slice */
107 unsigned int io_disp[2]; 117 unsigned int io_disp[2];
108 118
109 unsigned long last_low_overflow_time[2 119 unsigned long last_low_overflow_time[2];
110 120
111 uint64_t last_bytes_disp[2]; 121 uint64_t last_bytes_disp[2];
112 unsigned int last_io_disp[2]; 122 unsigned int last_io_disp[2];
113 123
114 /* <<
115 * The following two fields are update <<
116 * submitted while some bios are still <<
117 * bytes/ios are waited already in pre <<
118 * be used to calculate wait time unde <<
119 */ <<
120 long long carryover_bytes[2]; <<
121 int carryover_ios[2]; <<
122 <<
123 unsigned long last_check_time; 124 unsigned long last_check_time;
124 125
>> 126 unsigned long latency_target; /* us */
>> 127 unsigned long latency_target_conf; /* us */
125 /* When did we start a new slice */ 128 /* When did we start a new slice */
126 unsigned long slice_start[2]; 129 unsigned long slice_start[2];
127 unsigned long slice_end[2]; 130 unsigned long slice_end[2];
128 131
>> 132 unsigned long last_finish_time; /* ns / 1024 */
>> 133 unsigned long checked_last_finish_time; /* ns / 1024 */
>> 134 unsigned long avg_idletime; /* ns / 1024 */
>> 135 unsigned long idletime_threshold; /* us */
>> 136 unsigned long idletime_threshold_conf; /* us */
>> 137
>> 138 unsigned int bio_cnt; /* total bios */
>> 139 unsigned int bad_bio_cnt; /* bios exceeding latency threshold */
>> 140 unsigned long bio_cnt_reset_time;
>> 141
129 struct blkg_rwstat stat_bytes; 142 struct blkg_rwstat stat_bytes;
130 struct blkg_rwstat stat_ios; 143 struct blkg_rwstat stat_ios;
131 }; 144 };
132 145
133 extern struct blkcg_policy blkcg_policy_throtl 146 extern struct blkcg_policy blkcg_policy_throtl;
134 147
135 static inline struct throtl_grp *pd_to_tg(stru 148 static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
136 { 149 {
137 return pd ? container_of(pd, struct th 150 return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
138 } 151 }
139 152
140 static inline struct throtl_grp *blkg_to_tg(st 153 static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
141 { 154 {
142 return pd_to_tg(blkg_to_pd(blkg, &blkc 155 return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
143 } 156 }
144 157
145 /* 158 /*
146 * Internal throttling interface 159 * Internal throttling interface
147 */ 160 */
148 #ifndef CONFIG_BLK_DEV_THROTTLING 161 #ifndef CONFIG_BLK_DEV_THROTTLING
>> 162 static inline int blk_throtl_init(struct gendisk *disk) { return 0; }
149 static inline void blk_throtl_exit(struct gend 163 static inline void blk_throtl_exit(struct gendisk *disk) { }
>> 164 static inline void blk_throtl_register_queue(struct request_queue *q) { }
150 static inline bool blk_throtl_bio(struct bio * 165 static inline bool blk_throtl_bio(struct bio *bio) { return false; }
151 static inline void blk_throtl_cancel_bios(stru !! 166 static inline void blk_throtl_cancel_bios(struct request_queue *q) { }
152 #else /* CONFIG_BLK_DEV_THROTTLING */ 167 #else /* CONFIG_BLK_DEV_THROTTLING */
>> 168 int blk_throtl_init(struct gendisk *disk);
153 void blk_throtl_exit(struct gendisk *disk); 169 void blk_throtl_exit(struct gendisk *disk);
>> 170 void blk_throtl_register_queue(struct request_queue *q);
154 bool __blk_throtl_bio(struct bio *bio); 171 bool __blk_throtl_bio(struct bio *bio);
155 void blk_throtl_cancel_bios(struct gendisk *di !! 172 void blk_throtl_cancel_bios(struct request_queue *q);
156 !! 173 static inline bool blk_throtl_bio(struct bio *bio)
157 static inline bool blk_throtl_activated(struct <<
158 { <<
159 return q->td != NULL; <<
160 } <<
161 <<
162 static inline bool blk_should_throtl(struct bi <<
163 { 174 {
164 struct throtl_grp *tg; !! 175 struct throtl_grp *tg = blkg_to_tg(bio->bi_blkg);
165 int rw = bio_data_dir(bio); <<
166 176
167 /* !! 177 /* no need to throttle bps any more if the bio has been throttled */
168 * This is called under bio_queue_ente !! 178 if (bio_flagged(bio, BIO_BPS_THROTTLED) &&
169 * the activation of blk-throtl, which !! 179 !(tg->flags & THROTL_TG_HAS_IOPS_LIMIT))
170 * blk_mq_freeze_queue(). <<
171 */ <<
172 if (!blk_throtl_activated(bio->bi_bdev <<
173 return false; 180 return false;
174 181
175 tg = blkg_to_tg(bio->bi_blkg); !! 182 if (!tg->has_rules[bio_data_dir(bio)])
176 if (!cgroup_subsys_on_dfl(io_cgrp_subs <<
177 if (!bio_flagged(bio, BIO_CGRO <<
178 bio_set_flag(bio, BIO_ <<
179 blkg_rwstat_add(&tg->s <<
180 bio->b <<
181 } <<
182 blkg_rwstat_add(&tg->stat_ios, <<
183 } <<
184 <<
185 /* iops limit is always counted */ <<
186 if (tg->has_rules_iops[rw]) <<
187 return true; <<
188 <<
189 if (tg->has_rules_bps[rw] && !bio_flag <<
190 return true; <<
191 <<
192 return false; <<
193 } <<
194 <<
195 static inline bool blk_throtl_bio(struct bio * <<
196 { <<
197 <<
198 if (!blk_should_throtl(bio)) <<
199 return false; 183 return false;
200 184
201 return __blk_throtl_bio(bio); 185 return __blk_throtl_bio(bio);
202 } 186 }
203 #endif /* CONFIG_BLK_DEV_THROTTLING */ 187 #endif /* CONFIG_BLK_DEV_THROTTLING */
204 188
205 #endif 189 #endif
206 190
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.