[1/2] move wait ops' contention case completely out of line

From: William Lee Irwin III (wli_at_holomorphy.com)
Date: 08/31/04

  • Next message: Andrew Chew: "IDE class driver with SATA controllers" 
    Date:	Tue, 31 Aug 2004 14:03:46 -0700
To: Andrew Morton <akpm@osdl.org>

    On Tue, Aug 31, 2004 at 01:44:58AM -0700, William Lee Irwin III wrote:
    > Incremental atop the fastcall fix:
    > wait_on_bit_lock() needs to test_and_set_bit() in the fastpath, not
    > test_bit().

    Move the slow paths of wait_on_bit() and wait_on_bit_lock() out of line.
    Also uninline wake_up_bit() to reduce the number of callsites
    generated, and adjust loop startup in __wait_on_bit_lock() to properly
    reflect its usage in the contention case.

    Incremental atop the fastcall and wait_on_bit_lock()/test_and_set_bit() fixes.
    Successfully tested on x86-64.

    Index: mm2-2.6.9-rc1/include/linux/wait.h
    ===================================================================
    --- mm2-2.6.9-rc1.orig/include/linux/wait.h 2004-08-31 01:38:34.670946376 -0700
    +++ mm2-2.6.9-rc1/include/linux/wait.h 2004-08-31 01:50:40.947535552 -0700
    @@ -142,23 +142,11 @@
     void FASTCALL(__wake_up_bit(wait_queue_head_t *, void *, int));
     int FASTCALL(__wait_on_bit(wait_queue_head_t *, struct wait_bit_queue *, void *, int, int (*)(void *), unsigned));
     int FASTCALL(__wait_on_bit_lock(wait_queue_head_t *, struct wait_bit_queue *, void *, int, int (*)(void *), unsigned));
    +void FASTCALL(wake_up_bit(void *, int));
    +int FASTCALL(out_of_line_wait_on_bit(void *, int, int (*)(void *), unsigned));
    +int FASTCALL(out_of_line_wait_on_bit_lock(void *, int, int (*)(void *), unsigned));
     wait_queue_head_t *FASTCALL(bit_waitqueue(void *, int));
     
    -/**
    - * wake_up_bit - wake up a waiter on a bit
    - * @word: the word being waited on, a kernel virtual address
    - * @bit: the bit of the word being waited on
    - *
    - * There is a standard hashed waitqueue table for generic use. This
    - * is the part of the hashtable's accessor API that wakes up waiters
    - * on a bit. For instance, if one were to have waiters on a bitflag,
    - * one would call wake_up_bit() after clearing the bit.
    - */
    -static inline void wake_up_bit(void *word, int bit)
    -{
    - __wake_up_bit(bit_waitqueue(word, bit), word, bit);
    -}
    -
     #define wake_up(x) __wake_up(x, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, 1, NULL)
     #define wake_up_nr(x, nr) __wake_up(x, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, nr, NULL)
     #define wake_up_all(x) __wake_up(x, TASK_UNINTERRUPTIBLE | TASK_INTERRUPTIBLE, 0, NULL)
    @@ -356,14 +344,9 @@
     static inline int wait_on_bit(void *word, int bit,
                                     int (*action)(void *), unsigned mode)
     {
    - DEFINE_WAIT_BIT(q, word, bit);
    - wait_queue_head_t *wqh;
    -
             if (!test_bit(bit, word))
                     return 0;
    -
    - wqh = bit_waitqueue(word, bit);
    - return __wait_on_bit(wqh, &q, word, bit, action, mode);
    + return out_of_line_wait_on_bit(word, bit, action, mode);
     }
     
     /**
    @@ -385,14 +368,9 @@
     static inline int wait_on_bit_lock(void *word, int bit,
                                     int (*action)(void *), unsigned mode)
     {
    - DEFINE_WAIT_BIT(q, word, bit);
    - wait_queue_head_t *wqh;
    -
             if (!test_and_set_bit(bit, word))
                     return 0;
    -
    - wqh = bit_waitqueue(word, bit);
    - return __wait_on_bit_lock(wqh, &q, word, bit, action, mode);
    + return out_of_line_wait_on_bit_lock(word, bit, action, mode);
     }
             
     #endif /* __KERNEL__ */
    Index: mm2-2.6.9-rc1/kernel/wait.c
    ===================================================================
    --- mm2-2.6.9-rc1.orig/kernel/wait.c 2004-08-31 01:10:49.805044464 -0700
    +++ mm2-2.6.9-rc1/kernel/wait.c 2004-08-31 01:51:18.966755752 -0700
    @@ -164,24 +164,44 @@
     }
     EXPORT_SYMBOL(__wait_on_bit);
     
    +int __sched fastcall out_of_line_wait_on_bit(void *word, int bit,
    + int (*action)(void *), unsigned mode)
    +{
    + wait_queue_head_t *wq = bit_waitqueue(word, bit);
    + DEFINE_WAIT_BIT(wait, word, bit);
    +
    + return __wait_on_bit(wq, &wait, word, bit, action, mode);
    +}
    +EXPORT_SYMBOL(out_of_line_wait_on_bit);
    +
     int __sched fastcall __wait_on_bit_lock(wait_queue_head_t *wq,
                             struct wait_bit_queue *q, void *word, int bit,
                             int (*action)(void *), unsigned mode)
     {
             int ret = 0;
     
    - while (test_and_set_bit(bit, word)) {
    + do {
                     prepare_to_wait_exclusive(wq, &q->wait, mode);
                     if (test_bit(bit, word)) {
                             if ((ret = (*action)(word)))
                                     break;
                     }
    - }
    + } while (test_and_set_bit(bit, word));
             finish_wait(wq, &q->wait);
             return ret;
     }
     EXPORT_SYMBOL(__wait_on_bit_lock);
     
    +int __sched fastcall out_of_line_wait_on_bit_lock(void *word, int bit,
    + int (*action)(void *), unsigned mode)
    +{
    + wait_queue_head_t *wq = bit_waitqueue(word, bit);
    + DEFINE_WAIT_BIT(wait, word, bit);
    +
    + return __wait_on_bit_lock(wq, &wait, word, bit, action, mode);
    +}
    +EXPORT_SYMBOL(out_of_line_wait_on_bit_lock);
    +
     void fastcall __wake_up_bit(wait_queue_head_t *wq, void *word, int bit)
     {
             struct wait_bit_key key = __WAIT_BIT_KEY_INITIALIZER(word, bit);
    @@ -190,6 +210,22 @@
     }
     EXPORT_SYMBOL(__wake_up_bit);
     
    +/**
    + * wake_up_bit - wake up a waiter on a bit
    + * @word: the word being waited on, a kernel virtual address
    + * @bit: the bit of the word being waited on
    + *
    + * There is a standard hashed waitqueue table for generic use. This
    + * is the part of the hashtable's accessor API that wakes up waiters
    + * on a bit. For instance, if one were to have waiters on a bitflag,
    + * one would call wake_up_bit() after clearing the bit.
    + */
    +void fastcall wake_up_bit(void *word, int bit)
    +{
    + __wake_up_bit(bit_waitqueue(word, bit), word, bit);
    +}
    +EXPORT_SYMBOL(wake_up_bit);
    +
     wait_queue_head_t * fastcall bit_waitqueue(void *word, int bit)
     {
             const int shift = BITS_PER_LONG == 32 ? 5 : 6;
    -
    To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
    the body of a message to majordomo@vger.kernel.org
    More majordomo info at http://vger.kernel.org/majordomo-info.html
    Please read the FAQ at http://www.tux.org/lkml/

  • Next message: Andrew Chew: "IDE class driver with SATA controllers"

    Relevant Pages