1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0
2 2
>> 3 #include <linux/blk-mq.h>
3 #include <linux/blk-pm.h> 4 #include <linux/blk-pm.h>
4 #include <linux/blkdev.h> 5 #include <linux/blkdev.h>
5 #include <linux/pm_runtime.h> 6 #include <linux/pm_runtime.h>
6 #include "blk-mq.h" 7 #include "blk-mq.h"
>> 8 #include "blk-mq-tag.h"
7 9
8 /** 10 /**
9 * blk_pm_runtime_init - Block layer runtime P 11 * blk_pm_runtime_init - Block layer runtime PM initialization routine
10 * @q: the queue of the device 12 * @q: the queue of the device
11 * @dev: the device the queue belongs to 13 * @dev: the device the queue belongs to
12 * 14 *
13 * Description: 15 * Description:
14 * Initialize runtime-PM-related fields for 16 * Initialize runtime-PM-related fields for @q and start auto suspend for
15 * @dev. Drivers that want to take advantag 17 * @dev. Drivers that want to take advantage of request-based runtime PM
16 * should call this function after @dev has 18 * should call this function after @dev has been initialized, and its
17 * request queue @q has been allocated, and 19 * request queue @q has been allocated, and runtime PM for it can not happen
18 * yet(either due to disabled/forbidden or 20 * yet(either due to disabled/forbidden or its usage_count > 0). In most
19 * cases, driver should call this function 21 * cases, driver should call this function before any I/O has taken place.
20 * 22 *
21 * This function takes care of setting up u 23 * This function takes care of setting up using auto suspend for the device,
22 * the autosuspend delay is set to -1 to ma 24 * the autosuspend delay is set to -1 to make runtime suspend impossible
23 * until an updated value is either set by 25 * until an updated value is either set by user or by driver. Drivers do
24 * not need to touch other autosuspend sett 26 * not need to touch other autosuspend settings.
25 * 27 *
26 * The block layer runtime PM is request ba 28 * The block layer runtime PM is request based, so only works for drivers
27 * that use request as their IO unit instea 29 * that use request as their IO unit instead of those directly use bio's.
28 */ 30 */
29 void blk_pm_runtime_init(struct request_queue 31 void blk_pm_runtime_init(struct request_queue *q, struct device *dev)
30 { 32 {
31 q->dev = dev; 33 q->dev = dev;
32 q->rpm_status = RPM_ACTIVE; 34 q->rpm_status = RPM_ACTIVE;
33 pm_runtime_set_autosuspend_delay(q->de 35 pm_runtime_set_autosuspend_delay(q->dev, -1);
34 pm_runtime_use_autosuspend(q->dev); 36 pm_runtime_use_autosuspend(q->dev);
35 } 37 }
36 EXPORT_SYMBOL(blk_pm_runtime_init); 38 EXPORT_SYMBOL(blk_pm_runtime_init);
37 39
38 /** 40 /**
39 * blk_pre_runtime_suspend - Pre runtime suspe 41 * blk_pre_runtime_suspend - Pre runtime suspend check
40 * @q: the queue of the device 42 * @q: the queue of the device
41 * 43 *
42 * Description: 44 * Description:
43 * This function will check if runtime susp 45 * This function will check if runtime suspend is allowed for the device
44 * by examining if there are any requests p 46 * by examining if there are any requests pending in the queue. If there
45 * are requests pending, the device can not 47 * are requests pending, the device can not be runtime suspended; otherwise,
46 * the queue's status will be updated to SU 48 * the queue's status will be updated to SUSPENDING and the driver can
47 * proceed to suspend the device. 49 * proceed to suspend the device.
48 * 50 *
49 * For the not allowed case, we mark last b 51 * For the not allowed case, we mark last busy for the device so that
50 * runtime PM core will try to autosuspend 52 * runtime PM core will try to autosuspend it some time later.
51 * 53 *
52 * This function should be called near the 54 * This function should be called near the start of the device's
53 * runtime_suspend callback. 55 * runtime_suspend callback.
54 * 56 *
55 * Return: 57 * Return:
56 * 0 - OK to runtime suspend the de 58 * 0 - OK to runtime suspend the device
57 * -EBUSY - Device should not be runtime 59 * -EBUSY - Device should not be runtime suspended
58 */ 60 */
59 int blk_pre_runtime_suspend(struct request_que 61 int blk_pre_runtime_suspend(struct request_queue *q)
60 { 62 {
61 int ret = 0; 63 int ret = 0;
62 64
63 if (!q->dev) 65 if (!q->dev)
64 return ret; 66 return ret;
65 67
66 WARN_ON_ONCE(q->rpm_status != RPM_ACTI 68 WARN_ON_ONCE(q->rpm_status != RPM_ACTIVE);
67 69
68 spin_lock_irq(&q->queue_lock); <<
69 q->rpm_status = RPM_SUSPENDING; <<
70 spin_unlock_irq(&q->queue_lock); <<
71 <<
72 /* 70 /*
73 * Increase the pm_only counter before 71 * Increase the pm_only counter before checking whether any
74 * non-PM blk_queue_enter() calls are 72 * non-PM blk_queue_enter() calls are in progress to avoid that any
75 * new non-PM blk_queue_enter() calls 73 * new non-PM blk_queue_enter() calls succeed before the pm_only
76 * counter is decreased again. 74 * counter is decreased again.
77 */ 75 */
78 blk_set_pm_only(q); 76 blk_set_pm_only(q);
79 ret = -EBUSY; 77 ret = -EBUSY;
80 /* Switch q_usage_counter from per-cpu 78 /* Switch q_usage_counter from per-cpu to atomic mode. */
81 blk_freeze_queue_start(q); 79 blk_freeze_queue_start(q);
82 /* 80 /*
83 * Wait until atomic mode has been rea 81 * Wait until atomic mode has been reached. Since that
84 * involves calling call_rcu(), it is 82 * involves calling call_rcu(), it is guaranteed that later
85 * blk_queue_enter() calls see the pm- 83 * blk_queue_enter() calls see the pm-only state. See also
86 * http://lwn.net/Articles/573497/. 84 * http://lwn.net/Articles/573497/.
87 */ 85 */
88 percpu_ref_switch_to_atomic_sync(&q->q 86 percpu_ref_switch_to_atomic_sync(&q->q_usage_counter);
89 if (percpu_ref_is_zero(&q->q_usage_cou 87 if (percpu_ref_is_zero(&q->q_usage_counter))
90 ret = 0; 88 ret = 0;
91 /* Switch q_usage_counter back to per- 89 /* Switch q_usage_counter back to per-cpu mode. */
92 blk_mq_unfreeze_queue(q); 90 blk_mq_unfreeze_queue(q);
93 91
94 if (ret < 0) { !! 92 spin_lock_irq(&q->queue_lock);
95 spin_lock_irq(&q->queue_lock); !! 93 if (ret < 0)
96 q->rpm_status = RPM_ACTIVE; <<
97 pm_runtime_mark_last_busy(q->d 94 pm_runtime_mark_last_busy(q->dev);
98 spin_unlock_irq(&q->queue_lock !! 95 else
>> 96 q->rpm_status = RPM_SUSPENDING;
>> 97 spin_unlock_irq(&q->queue_lock);
99 98
>> 99 if (ret)
100 blk_clear_pm_only(q); 100 blk_clear_pm_only(q);
101 } <<
102 101
103 return ret; 102 return ret;
104 } 103 }
105 EXPORT_SYMBOL(blk_pre_runtime_suspend); 104 EXPORT_SYMBOL(blk_pre_runtime_suspend);
106 105
107 /** 106 /**
108 * blk_post_runtime_suspend - Post runtime sus 107 * blk_post_runtime_suspend - Post runtime suspend processing
109 * @q: the queue of the device 108 * @q: the queue of the device
110 * @err: return value of the device's runtime_ 109 * @err: return value of the device's runtime_suspend function
111 * 110 *
112 * Description: 111 * Description:
113 * Update the queue's runtime status accord 112 * Update the queue's runtime status according to the return value of the
114 * device's runtime suspend function and ma 113 * device's runtime suspend function and mark last busy for the device so
115 * that PM core will try to auto suspend th 114 * that PM core will try to auto suspend the device at a later time.
116 * 115 *
117 * This function should be called near the 116 * This function should be called near the end of the device's
118 * runtime_suspend callback. 117 * runtime_suspend callback.
119 */ 118 */
120 void blk_post_runtime_suspend(struct request_q 119 void blk_post_runtime_suspend(struct request_queue *q, int err)
121 { 120 {
122 if (!q->dev) 121 if (!q->dev)
123 return; 122 return;
124 123
125 spin_lock_irq(&q->queue_lock); 124 spin_lock_irq(&q->queue_lock);
126 if (!err) { 125 if (!err) {
127 q->rpm_status = RPM_SUSPENDED; 126 q->rpm_status = RPM_SUSPENDED;
128 } else { 127 } else {
129 q->rpm_status = RPM_ACTIVE; 128 q->rpm_status = RPM_ACTIVE;
130 pm_runtime_mark_last_busy(q->d 129 pm_runtime_mark_last_busy(q->dev);
131 } 130 }
132 spin_unlock_irq(&q->queue_lock); 131 spin_unlock_irq(&q->queue_lock);
133 132
134 if (err) 133 if (err)
135 blk_clear_pm_only(q); 134 blk_clear_pm_only(q);
136 } 135 }
137 EXPORT_SYMBOL(blk_post_runtime_suspend); 136 EXPORT_SYMBOL(blk_post_runtime_suspend);
138 137
139 /** 138 /**
140 * blk_pre_runtime_resume - Pre runtime resume 139 * blk_pre_runtime_resume - Pre runtime resume processing
141 * @q: the queue of the device 140 * @q: the queue of the device
142 * 141 *
143 * Description: 142 * Description:
144 * Update the queue's runtime status to RES 143 * Update the queue's runtime status to RESUMING in preparation for the
145 * runtime resume of the device. 144 * runtime resume of the device.
146 * 145 *
147 * This function should be called near the 146 * This function should be called near the start of the device's
148 * runtime_resume callback. 147 * runtime_resume callback.
149 */ 148 */
150 void blk_pre_runtime_resume(struct request_que 149 void blk_pre_runtime_resume(struct request_queue *q)
151 { 150 {
152 if (!q->dev) 151 if (!q->dev)
153 return; 152 return;
154 153
155 spin_lock_irq(&q->queue_lock); 154 spin_lock_irq(&q->queue_lock);
156 q->rpm_status = RPM_RESUMING; 155 q->rpm_status = RPM_RESUMING;
157 spin_unlock_irq(&q->queue_lock); 156 spin_unlock_irq(&q->queue_lock);
158 } 157 }
159 EXPORT_SYMBOL(blk_pre_runtime_resume); 158 EXPORT_SYMBOL(blk_pre_runtime_resume);
160 159
161 /** 160 /**
162 * blk_post_runtime_resume - Post runtime resu 161 * blk_post_runtime_resume - Post runtime resume processing
163 * @q: the queue of the device 162 * @q: the queue of the device
>> 163 * @err: return value of the device's runtime_resume function
164 * 164 *
165 * Description: 165 * Description:
166 * Restart the queue of a runtime suspended !! 166 * Update the queue's runtime status according to the return value of the
167 * of whether the device's runtime-resume s !! 167 * device's runtime_resume function. If it is successfully resumed, process
168 * driver or error handler will need to com !! 168 * the requests that are queued into the device's queue when it is resuming
>> 169 * and then mark last busy and initiate autosuspend for it.
169 * 170 *
170 * This function should be called near the 171 * This function should be called near the end of the device's
171 * runtime_resume callback to correct queue !! 172 * runtime_resume callback.
172 * peeking requests from the queue. <<
173 */ 173 */
174 void blk_post_runtime_resume(struct request_qu !! 174 void blk_post_runtime_resume(struct request_queue *q, int err)
175 { 175 {
176 int old_status; <<
177 <<
178 if (!q->dev) 176 if (!q->dev)
179 return; 177 return;
180 178
181 spin_lock_irq(&q->queue_lock); 179 spin_lock_irq(&q->queue_lock);
182 old_status = q->rpm_status; !! 180 if (!err) {
183 q->rpm_status = RPM_ACTIVE; !! 181 q->rpm_status = RPM_ACTIVE;
184 pm_runtime_mark_last_busy(q->dev); !! 182 pm_runtime_mark_last_busy(q->dev);
185 pm_request_autosuspend(q->dev); !! 183 pm_request_autosuspend(q->dev);
>> 184 } else {
>> 185 q->rpm_status = RPM_SUSPENDED;
>> 186 }
186 spin_unlock_irq(&q->queue_lock); 187 spin_unlock_irq(&q->queue_lock);
187 188
188 if (old_status != RPM_ACTIVE) !! 189 if (!err)
189 blk_clear_pm_only(q); 190 blk_clear_pm_only(q);
190 } 191 }
191 EXPORT_SYMBOL(blk_post_runtime_resume); 192 EXPORT_SYMBOL(blk_post_runtime_resume);
>> 193
>> 194 /**
>> 195 * blk_set_runtime_active - Force runtime status of the queue to be active
>> 196 * @q: the queue of the device
>> 197 *
>> 198 * If the device is left runtime suspended during system suspend the resume
>> 199 * hook typically resumes the device and corrects runtime status
>> 200 * accordingly. However, that does not affect the queue runtime PM status
>> 201 * which is still "suspended". This prevents processing requests from the
>> 202 * queue.
>> 203 *
>> 204 * This function can be used in driver's resume hook to correct queue
>> 205 * runtime PM status and re-enable peeking requests from the queue. It
>> 206 * should be called before first request is added to the queue.
>> 207 */
>> 208 void blk_set_runtime_active(struct request_queue *q)
>> 209 {
>> 210 if (q->dev) {
>> 211 spin_lock_irq(&q->queue_lock);
>> 212 q->rpm_status = RPM_ACTIVE;
>> 213 pm_runtime_mark_last_busy(q->dev);
>> 214 pm_request_autosuspend(q->dev);
>> 215 spin_unlock_irq(&q->queue_lock);
>> 216 }
>> 217 }
>> 218 EXPORT_SYMBOL(blk_set_runtime_active);
192 219
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.