From 92127c7a45d4d167d9b015a5f9de6b41ed66f1d0 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Sun, 26 Mar 2006 01:38:05 -0800 Subject: [PATCH] [PATCH] hrtimers: optimize softirq runqueues The hrtimer softirq is called from the timer softirq every tick. Retrieve the current time from xtime and wall_to_monotonic instead of calling base->get_time() for each timer base. Store the time in the base structure and provide a hook once clock source abstractions are in place and to keep the code open for new base clocks. Based on a patch from: Roman Zippel Signed-off-by: Thomas Gleixner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/hrtimer.h | 20 ++++++++++++-------- kernel/hrtimer.c | 28 ++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 6401c31d6ad..64e2754ca73 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -72,14 +72,16 @@ struct hrtimer { /** * struct hrtimer_base - the timer base for a specific clock * - * @index: clock type index for per_cpu support when moving a timer - * to a base on another cpu. - * @lock: lock protecting the base and associated timers - * @active: red black tree root node for the active timers - * @first: pointer to the timer node which expires first - * @resolution: the resolution of the clock, in nanoseconds - * @get_time: function to retrieve the current time of the clock - * @curr_timer: the timer which is executing a callback right now + * @index: clock type index for per_cpu support when moving a timer + * to a base on another cpu. + * @lock: lock protecting the base and associated timers + * @active: red black tree root node for the active timers + * @first: pointer to the timer node which expires first + * @resolution: the resolution of the clock, in nanoseconds + * @get_time: function to retrieve the current time of the clock + * @get_sofirq_time: function to retrieve the current time from the softirq + * @curr_timer: the timer which is executing a callback right now + * @softirq_time: the time when running the hrtimer queue in the softirq */ struct hrtimer_base { clockid_t index; @@ -88,7 +90,9 @@ struct hrtimer_base { struct rb_node *first; ktime_t resolution; ktime_t (*get_time)(void); + ktime_t (*get_softirq_time)(void); struct hrtimer *curr_timer; + ktime_t softirq_time; }; /* diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c index 14bc9cfa639..b728cc53452 100644 --- a/kernel/hrtimer.c +++ b/kernel/hrtimer.c @@ -122,6 +122,26 @@ void ktime_get_ts(struct timespec *ts) } EXPORT_SYMBOL_GPL(ktime_get_ts); +/* + * Get the coarse grained time at the softirq based on xtime and + * wall_to_monotonic. + */ +static void hrtimer_get_softirq_time(struct hrtimer_base *base) +{ + ktime_t xtim, tomono; + unsigned long seq; + + do { + seq = read_seqbegin(&xtime_lock); + xtim = timespec_to_ktime(xtime); + tomono = timespec_to_ktime(wall_to_monotonic); + + } while (read_seqretry(&xtime_lock, seq)); + + base[CLOCK_REALTIME].softirq_time = xtim; + base[CLOCK_MONOTONIC].softirq_time = ktime_add(xtim, tomono); +} + /* * Functions and macros which are different for UP/SMP systems are kept in a * single place @@ -586,9 +606,11 @@ int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp) */ static inline void run_hrtimer_queue(struct hrtimer_base *base) { - ktime_t now = base->get_time(); struct rb_node *node; + if (base->get_softirq_time) + base->softirq_time = base->get_softirq_time(); + spin_lock_irq(&base->lock); while ((node = base->first)) { @@ -598,7 +620,7 @@ static inline void run_hrtimer_queue(struct hrtimer_base *base) void *data; timer = rb_entry(node, struct hrtimer, node); - if (now.tv64 <= timer->expires.tv64) + if (base->softirq_time.tv64 <= timer->expires.tv64) break; fn = timer->function; @@ -641,6 +663,8 @@ void hrtimer_run_queues(void) struct hrtimer_base *base = __get_cpu_var(hrtimer_bases); int i; + hrtimer_get_softirq_time(base); + for (i = 0; i < MAX_HRTIMER_BASES; i++) run_hrtimer_queue(&base[i]); }