Avoid taking waitqueue lock in dmapool
With one trivial change (taking the lock slightly earlier on wakeup
from schedule), all uses of the waitq are under the pool lock, so we
can use the locked (or __) versions of the wait queue functions, and
avoid the extra spinlock.
Signed-off-by: Matthew Wilcox <willy@linux.intel.com>
Acked-by: David S. Miller <davem@davemloft.net>
diff --git a/mm/dmapool.c b/mm/dmapool.c
index 92e886d..b5ff9ce 100644
--- a/mm/dmapool.c
+++ b/mm/dmapool.c
@@ -275,8 +275,8 @@
size_t offset;
void *retval;
- restart:
spin_lock_irqsave(&pool->lock, flags);
+ restart:
list_for_each_entry(page, &pool->page_list, page_list) {
int i;
/* only cachable accesses here ... */
@@ -299,12 +299,13 @@
DECLARE_WAITQUEUE(wait, current);
__set_current_state(TASK_INTERRUPTIBLE);
- add_wait_queue(&pool->waitq, &wait);
+ __add_wait_queue(&pool->waitq, &wait);
spin_unlock_irqrestore(&pool->lock, flags);
schedule_timeout(POOL_TIMEOUT_JIFFIES);
- remove_wait_queue(&pool->waitq, &wait);
+ spin_lock_irqsave(&pool->lock, flags);
+ __remove_wait_queue(&pool->waitq, &wait);
goto restart;
}
retval = NULL;
@@ -406,7 +407,7 @@
page->in_use--;
set_bit(block, &page->bitmap[map]);
if (waitqueue_active(&pool->waitq))
- wake_up(&pool->waitq);
+ wake_up_locked(&pool->waitq);
/*
* Resist a temptation to do
* if (!is_page_busy(bpp, page->bitmap)) pool_free_page(pool, page);