1 // SPDX-License-Identifier: GPL-2.0 1 // SPDX-License-Identifier: GPL-2.0 2 2 >> 3 #include <linux/blk-mq.h> 3 #include <linux/blk-pm.h> 4 #include <linux/blk-pm.h> 4 #include <linux/blkdev.h> 5 #include <linux/blkdev.h> 5 #include <linux/pm_runtime.h> 6 #include <linux/pm_runtime.h> 6 #include "blk-mq.h" 7 #include "blk-mq.h" >> 8 #include "blk-mq-tag.h" 7 9 8 /** 10 /** 9 * blk_pm_runtime_init - Block layer runtime P 11 * blk_pm_runtime_init - Block layer runtime PM initialization routine 10 * @q: the queue of the device 12 * @q: the queue of the device 11 * @dev: the device the queue belongs to 13 * @dev: the device the queue belongs to 12 * 14 * 13 * Description: 15 * Description: 14 * Initialize runtime-PM-related fields for 16 * Initialize runtime-PM-related fields for @q and start auto suspend for 15 * @dev. Drivers that want to take advantag 17 * @dev. Drivers that want to take advantage of request-based runtime PM 16 * should call this function after @dev has 18 * should call this function after @dev has been initialized, and its 17 * request queue @q has been allocated, and 19 * request queue @q has been allocated, and runtime PM for it can not happen 18 * yet(either due to disabled/forbidden or 20 * yet(either due to disabled/forbidden or its usage_count > 0). In most 19 * cases, driver should call this function 21 * cases, driver should call this function before any I/O has taken place. 20 * 22 * 21 * This function takes care of setting up u 23 * This function takes care of setting up using auto suspend for the device, 22 * the autosuspend delay is set to -1 to ma 24 * the autosuspend delay is set to -1 to make runtime suspend impossible 23 * until an updated value is either set by 25 * until an updated value is either set by user or by driver. Drivers do 24 * not need to touch other autosuspend sett 26 * not need to touch other autosuspend settings. 25 * 27 * 26 * The block layer runtime PM is request ba 28 * The block layer runtime PM is request based, so only works for drivers 27 * that use request as their IO unit instea 29 * that use request as their IO unit instead of those directly use bio's. 28 */ 30 */ 29 void blk_pm_runtime_init(struct request_queue 31 void blk_pm_runtime_init(struct request_queue *q, struct device *dev) 30 { 32 { 31 q->dev = dev; 33 q->dev = dev; 32 q->rpm_status = RPM_ACTIVE; 34 q->rpm_status = RPM_ACTIVE; 33 pm_runtime_set_autosuspend_delay(q->de 35 pm_runtime_set_autosuspend_delay(q->dev, -1); 34 pm_runtime_use_autosuspend(q->dev); 36 pm_runtime_use_autosuspend(q->dev); 35 } 37 } 36 EXPORT_SYMBOL(blk_pm_runtime_init); 38 EXPORT_SYMBOL(blk_pm_runtime_init); 37 39 38 /** 40 /** 39 * blk_pre_runtime_suspend - Pre runtime suspe 41 * blk_pre_runtime_suspend - Pre runtime suspend check 40 * @q: the queue of the device 42 * @q: the queue of the device 41 * 43 * 42 * Description: 44 * Description: 43 * This function will check if runtime susp 45 * This function will check if runtime suspend is allowed for the device 44 * by examining if there are any requests p 46 * by examining if there are any requests pending in the queue. If there 45 * are requests pending, the device can not 47 * are requests pending, the device can not be runtime suspended; otherwise, 46 * the queue's status will be updated to SU 48 * the queue's status will be updated to SUSPENDING and the driver can 47 * proceed to suspend the device. 49 * proceed to suspend the device. 48 * 50 * 49 * For the not allowed case, we mark last b 51 * For the not allowed case, we mark last busy for the device so that 50 * runtime PM core will try to autosuspend 52 * runtime PM core will try to autosuspend it some time later. 51 * 53 * 52 * This function should be called near the 54 * This function should be called near the start of the device's 53 * runtime_suspend callback. 55 * runtime_suspend callback. 54 * 56 * 55 * Return: 57 * Return: 56 * 0 - OK to runtime suspend the de 58 * 0 - OK to runtime suspend the device 57 * -EBUSY - Device should not be runtime 59 * -EBUSY - Device should not be runtime suspended 58 */ 60 */ 59 int blk_pre_runtime_suspend(struct request_que 61 int blk_pre_runtime_suspend(struct request_queue *q) 60 { 62 { 61 int ret = 0; 63 int ret = 0; 62 64 63 if (!q->dev) 65 if (!q->dev) 64 return ret; 66 return ret; 65 67 66 WARN_ON_ONCE(q->rpm_status != RPM_ACTI 68 WARN_ON_ONCE(q->rpm_status != RPM_ACTIVE); 67 69 68 spin_lock_irq(&q->queue_lock); 70 spin_lock_irq(&q->queue_lock); 69 q->rpm_status = RPM_SUSPENDING; 71 q->rpm_status = RPM_SUSPENDING; 70 spin_unlock_irq(&q->queue_lock); 72 spin_unlock_irq(&q->queue_lock); 71 73 72 /* 74 /* 73 * Increase the pm_only counter before 75 * Increase the pm_only counter before checking whether any 74 * non-PM blk_queue_enter() calls are 76 * non-PM blk_queue_enter() calls are in progress to avoid that any 75 * new non-PM blk_queue_enter() calls 77 * new non-PM blk_queue_enter() calls succeed before the pm_only 76 * counter is decreased again. 78 * counter is decreased again. 77 */ 79 */ 78 blk_set_pm_only(q); 80 blk_set_pm_only(q); 79 ret = -EBUSY; 81 ret = -EBUSY; 80 /* Switch q_usage_counter from per-cpu 82 /* Switch q_usage_counter from per-cpu to atomic mode. */ 81 blk_freeze_queue_start(q); 83 blk_freeze_queue_start(q); 82 /* 84 /* 83 * Wait until atomic mode has been rea 85 * Wait until atomic mode has been reached. Since that 84 * involves calling call_rcu(), it is 86 * involves calling call_rcu(), it is guaranteed that later 85 * blk_queue_enter() calls see the pm- 87 * blk_queue_enter() calls see the pm-only state. See also 86 * http://lwn.net/Articles/573497/. 88 * http://lwn.net/Articles/573497/. 87 */ 89 */ 88 percpu_ref_switch_to_atomic_sync(&q->q 90 percpu_ref_switch_to_atomic_sync(&q->q_usage_counter); 89 if (percpu_ref_is_zero(&q->q_usage_cou 91 if (percpu_ref_is_zero(&q->q_usage_counter)) 90 ret = 0; 92 ret = 0; 91 /* Switch q_usage_counter back to per- 93 /* Switch q_usage_counter back to per-cpu mode. */ 92 blk_mq_unfreeze_queue(q); 94 blk_mq_unfreeze_queue(q); 93 95 94 if (ret < 0) { 96 if (ret < 0) { 95 spin_lock_irq(&q->queue_lock); 97 spin_lock_irq(&q->queue_lock); 96 q->rpm_status = RPM_ACTIVE; 98 q->rpm_status = RPM_ACTIVE; 97 pm_runtime_mark_last_busy(q->d 99 pm_runtime_mark_last_busy(q->dev); 98 spin_unlock_irq(&q->queue_lock 100 spin_unlock_irq(&q->queue_lock); 99 101 100 blk_clear_pm_only(q); 102 blk_clear_pm_only(q); 101 } 103 } 102 104 103 return ret; 105 return ret; 104 } 106 } 105 EXPORT_SYMBOL(blk_pre_runtime_suspend); 107 EXPORT_SYMBOL(blk_pre_runtime_suspend); 106 108 107 /** 109 /** 108 * blk_post_runtime_suspend - Post runtime sus 110 * blk_post_runtime_suspend - Post runtime suspend processing 109 * @q: the queue of the device 111 * @q: the queue of the device 110 * @err: return value of the device's runtime_ 112 * @err: return value of the device's runtime_suspend function 111 * 113 * 112 * Description: 114 * Description: 113 * Update the queue's runtime status accord 115 * Update the queue's runtime status according to the return value of the 114 * device's runtime suspend function and ma 116 * device's runtime suspend function and mark last busy for the device so 115 * that PM core will try to auto suspend th 117 * that PM core will try to auto suspend the device at a later time. 116 * 118 * 117 * This function should be called near the 119 * This function should be called near the end of the device's 118 * runtime_suspend callback. 120 * runtime_suspend callback. 119 */ 121 */ 120 void blk_post_runtime_suspend(struct request_q 122 void blk_post_runtime_suspend(struct request_queue *q, int err) 121 { 123 { 122 if (!q->dev) 124 if (!q->dev) 123 return; 125 return; 124 126 125 spin_lock_irq(&q->queue_lock); 127 spin_lock_irq(&q->queue_lock); 126 if (!err) { 128 if (!err) { 127 q->rpm_status = RPM_SUSPENDED; 129 q->rpm_status = RPM_SUSPENDED; 128 } else { 130 } else { 129 q->rpm_status = RPM_ACTIVE; 131 q->rpm_status = RPM_ACTIVE; 130 pm_runtime_mark_last_busy(q->d 132 pm_runtime_mark_last_busy(q->dev); 131 } 133 } 132 spin_unlock_irq(&q->queue_lock); 134 spin_unlock_irq(&q->queue_lock); 133 135 134 if (err) 136 if (err) 135 blk_clear_pm_only(q); 137 blk_clear_pm_only(q); 136 } 138 } 137 EXPORT_SYMBOL(blk_post_runtime_suspend); 139 EXPORT_SYMBOL(blk_post_runtime_suspend); 138 140 139 /** 141 /** 140 * blk_pre_runtime_resume - Pre runtime resume 142 * blk_pre_runtime_resume - Pre runtime resume processing 141 * @q: the queue of the device 143 * @q: the queue of the device 142 * 144 * 143 * Description: 145 * Description: 144 * Update the queue's runtime status to RES 146 * Update the queue's runtime status to RESUMING in preparation for the 145 * runtime resume of the device. 147 * runtime resume of the device. 146 * 148 * 147 * This function should be called near the 149 * This function should be called near the start of the device's 148 * runtime_resume callback. 150 * runtime_resume callback. 149 */ 151 */ 150 void blk_pre_runtime_resume(struct request_que 152 void blk_pre_runtime_resume(struct request_queue *q) 151 { 153 { 152 if (!q->dev) 154 if (!q->dev) 153 return; 155 return; 154 156 155 spin_lock_irq(&q->queue_lock); 157 spin_lock_irq(&q->queue_lock); 156 q->rpm_status = RPM_RESUMING; 158 q->rpm_status = RPM_RESUMING; 157 spin_unlock_irq(&q->queue_lock); 159 spin_unlock_irq(&q->queue_lock); 158 } 160 } 159 EXPORT_SYMBOL(blk_pre_runtime_resume); 161 EXPORT_SYMBOL(blk_pre_runtime_resume); 160 162 161 /** 163 /** 162 * blk_post_runtime_resume - Post runtime resu 164 * blk_post_runtime_resume - Post runtime resume processing 163 * @q: the queue of the device 165 * @q: the queue of the device 164 * 166 * 165 * Description: 167 * Description: 166 * Restart the queue of a runtime suspended !! 168 * For historical reasons, this routine merely calls blk_set_runtime_active() 167 * of whether the device's runtime-resume s !! 169 * to do the real work of restarting the queue. It does this regardless of >> 170 * whether the device's runtime-resume succeeded; even if it failed the 168 * driver or error handler will need to com 171 * driver or error handler will need to communicate with the device. 169 * 172 * 170 * This function should be called near the 173 * This function should be called near the end of the device's 171 * runtime_resume callback to correct queue !! 174 * runtime_resume callback. 172 * peeking requests from the queue. << 173 */ 175 */ 174 void blk_post_runtime_resume(struct request_qu 176 void blk_post_runtime_resume(struct request_queue *q) 175 { 177 { >> 178 blk_set_runtime_active(q); >> 179 } >> 180 EXPORT_SYMBOL(blk_post_runtime_resume); >> 181 >> 182 /** >> 183 * blk_set_runtime_active - Force runtime status of the queue to be active >> 184 * @q: the queue of the device >> 185 * >> 186 * If the device is left runtime suspended during system suspend the resume >> 187 * hook typically resumes the device and corrects runtime status >> 188 * accordingly. However, that does not affect the queue runtime PM status >> 189 * which is still "suspended". This prevents processing requests from the >> 190 * queue. >> 191 * >> 192 * This function can be used in driver's resume hook to correct queue >> 193 * runtime PM status and re-enable peeking requests from the queue. It >> 194 * should be called before first request is added to the queue. >> 195 * >> 196 * This function is also called by blk_post_runtime_resume() for >> 197 * runtime resumes. It does everything necessary to restart the queue. >> 198 */ >> 199 void blk_set_runtime_active(struct request_queue *q) >> 200 { 176 int old_status; 201 int old_status; 177 202 178 if (!q->dev) 203 if (!q->dev) 179 return; 204 return; 180 205 181 spin_lock_irq(&q->queue_lock); 206 spin_lock_irq(&q->queue_lock); 182 old_status = q->rpm_status; 207 old_status = q->rpm_status; 183 q->rpm_status = RPM_ACTIVE; 208 q->rpm_status = RPM_ACTIVE; 184 pm_runtime_mark_last_busy(q->dev); 209 pm_runtime_mark_last_busy(q->dev); 185 pm_request_autosuspend(q->dev); 210 pm_request_autosuspend(q->dev); 186 spin_unlock_irq(&q->queue_lock); 211 spin_unlock_irq(&q->queue_lock); 187 212 188 if (old_status != RPM_ACTIVE) 213 if (old_status != RPM_ACTIVE) 189 blk_clear_pm_only(q); 214 blk_clear_pm_only(q); 190 } 215 } 191 EXPORT_SYMBOL(blk_post_runtime_resume); !! 216 EXPORT_SYMBOL(blk_set_runtime_active); 192 217
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.