adulau/aha
1
0
Fork 0
mirror of https://github.com/adulau/aha.git synced 2024-12-28 19:56:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

cpumask: convert RCU implementations

Browse source 
Impact: use new cpumask API.

rcu_ctrlblk contains a cpumask, and it's highly optimized so I don't want
a cpumask_var_t (ie. a pointer) for the CONFIG_CPUMASK_OFFSTACK case.  It
could use a dangling bitmap, and be allocated in __rcu_init to save memory,
but for the moment we use a bitmap.

(Eventually 'struct cpumask' will be undefined for CONFIG_CPUMASK_OFFSTACK,
so we use a bitmap here to show we really mean it).

We remove on-stack cpumasks, using cpumask_var_t for
rcu_torture_shuffle_tasks() and for_each_cpu_and in force_quiescent_state().

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
This commit is contained in:
Rusty Russell 2009-01-01 10:12:26 +10:30
parent d036e67b40
commit bd232f97b3
4 changed files with 44 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
include/linux/rcuclassic.h
View file

@ -59,8 +59,8 @@ struct rcu_ctrlblk {
int signaled; int signaled;
spinlock_t lock ____cacheline_internodealigned_in_smp; spinlock_t lock ____cacheline_internodealigned_in_smp;
cpumask_t cpumask; /* CPUs that need to switch in order */ DECLARE_BITMAP(cpumask, NR_CPUS); /* CPUs that need to switch for */
/* for current batch to proceed. */ /* current batch to proceed. */
} ____cacheline_internodealigned_in_smp; } ____cacheline_internodealigned_in_smp;
/* Is batch a before batch b ? */ /* Is batch a before batch b ? */

30
kernel/rcuclassic.c
View file

@ -63,14 +63,14 @@ static struct rcu_ctrlblk rcu_ctrlblk = {
.completed = -300, .completed = -300,
.pending = -300, .pending = -300,
.lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock), .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock),
.cpumask = CPU_MASK_NONE, .cpumask = CPU_BITS_NONE,
}; };
static struct rcu_ctrlblk rcu_bh_ctrlblk = { static struct rcu_ctrlblk rcu_bh_ctrlblk = {
.cur = -300, .cur = -300,
.completed = -300, .completed = -300,
.pending = -300, .pending = -300,
.lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock), .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock),
.cpumask = CPU_MASK_NONE, .cpumask = CPU_BITS_NONE,
}; };
DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L }; DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L };
@ -85,7 +85,6 @@ static void force_quiescent_state(struct rcu_data *rdp,
struct rcu_ctrlblk *rcp) struct rcu_ctrlblk *rcp)
{ {
int cpu; int cpu;
cpumask_t cpumask;
unsigned long flags; unsigned long flags;
set_need_resched(); set_need_resched();
@ -96,10 +95,10 @@ static void force_quiescent_state(struct rcu_data *rdp,
* Don't send IPI to itself. With irqs disabled, * Don't send IPI to itself. With irqs disabled,
* rdp->cpu is the current cpu. * rdp->cpu is the current cpu.
* *
* cpu_online_map is updated by the _cpu_down() * cpu_online_mask is updated by the _cpu_down()
* using __stop_machine(). Since we're in irqs disabled * using __stop_machine(). Since we're in irqs disabled
* section, __stop_machine() is not exectuting, hence * section, __stop_machine() is not exectuting, hence
* the cpu_online_map is stable. * the cpu_online_mask is stable.
* *
* However, a cpu might have been offlined _just_ before * However, a cpu might have been offlined _just_ before
* we disabled irqs while entering here. * we disabled irqs while entering here.
@ -107,14 +106,15 @@ static void force_quiescent_state(struct rcu_data *rdp,
* notification, leading to the offlined cpu's bit * notification, leading to the offlined cpu's bit
* being set in the rcp->cpumask. * being set in the rcp->cpumask.
* *
* Hence cpumask = (rcp->cpumask & cpu_online_map) to prevent * Hence cpumask = (rcp->cpumask & cpu_online_mask) to prevent
* sending smp_reschedule() to an offlined CPU. * sending smp_reschedule() to an offlined CPU.
*/ */
cpus_and(cpumask, rcp->cpumask, cpu_online_map); for_each_cpu_and(cpu,
cpu_clear(rdp->cpu, cpumask); to_cpumask(rcp->cpumask), cpu_online_mask) {
for_each_cpu_mask_nr(cpu, cpumask) if (cpu != rdp->cpu)
smp_send_reschedule(cpu); smp_send_reschedule(cpu);
} }
}
spin_unlock_irqrestore(&rcp->lock, flags); spin_unlock_irqrestore(&rcp->lock, flags);
} }
#else #else
@ -193,7 +193,7 @@ static void print_other_cpu_stall(struct rcu_ctrlblk *rcp)
printk(KERN_ERR "INFO: RCU detected CPU stalls:"); printk(KERN_ERR "INFO: RCU detected CPU stalls:");
for_each_possible_cpu(cpu) { for_each_possible_cpu(cpu) {
if (cpu_isset(cpu, rcp->cpumask)) if (cpumask_test_cpu(cpu, to_cpumask(rcp->cpumask)))
printk(" %d", cpu); printk(" %d", cpu);
} }
printk(" (detected by %d, t=%ld jiffies)\n", printk(" (detected by %d, t=%ld jiffies)\n",
@ -221,7 +221,8 @@ static void check_cpu_stall(struct rcu_ctrlblk *rcp)
long delta; long delta;
delta = jiffies - rcp->jiffies_stall; delta = jiffies - rcp->jiffies_stall;
if (cpu_isset(smp_processor_id(), rcp->cpumask) && delta >= 0) { if (cpumask_test_cpu(smp_processor_id(), to_cpumask(rcp->cpumask)) &&
delta >= 0) {
/* We haven't checked in, so go dump stack. */ /* We haven't checked in, so go dump stack. */
print_cpu_stall(rcp); print_cpu_stall(rcp);
@ -393,7 +394,8 @@ static void rcu_start_batch(struct rcu_ctrlblk *rcp)
* unnecessarily. * unnecessarily.
*/ */
smp_mb(); smp_mb();
cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask); cpumask_andnot(to_cpumask(rcp->cpumask),
cpu_online_mask, &nohz_cpu_mask);
rcp->signaled = 0; rcp->signaled = 0;
} }
@ -406,8 +408,8 @@ static void rcu_start_batch(struct rcu_ctrlblk *rcp)
*/ */
static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp) static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp)
{ {
cpu_clear(cpu, rcp->cpumask); cpumask_clear_cpu(cpu, to_cpumask(rcp->cpumask));
if (cpus_empty(rcp->cpumask)) { if (cpumask_empty(to_cpumask(rcp->cpumask))) {
/* batch completed ! */ /* batch completed ! */
rcp->completed = rcp->cur; rcp->completed = rcp->cur;
rcu_start_batch(rcp); rcu_start_batch(rcp);

19
kernel/rcupreempt.c
View file

@ -164,7 +164,8 @@ static char *rcu_try_flip_state_names[] =
{ "idle", "waitack", "waitzero", "waitmb" }; { "idle", "waitack", "waitzero", "waitmb" };
#endif /* #ifdef CONFIG_RCU_TRACE */ #endif /* #ifdef CONFIG_RCU_TRACE */
static cpumask_t rcu_cpu_online_map __read_mostly = CPU_MASK_NONE; static DECLARE_BITMAP(rcu_cpu_online_map, NR_CPUS) __read_mostly
= CPU_BITS_NONE;
/* /*
* Enum and per-CPU flag to determine when each CPU has seen * Enum and per-CPU flag to determine when each CPU has seen
@ -758,7 +759,7 @@ rcu_try_flip_idle(void)
/* Now ask each CPU for acknowledgement of the flip. */ /* Now ask each CPU for acknowledgement of the flip. */
for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) { for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {
per_cpu(rcu_flip_flag, cpu) = rcu_flipped; per_cpu(rcu_flip_flag, cpu) = rcu_flipped;
dyntick_save_progress_counter(cpu); dyntick_save_progress_counter(cpu);
} }
@ -776,7 +777,7 @@ rcu_try_flip_waitack(void)
int cpu; int cpu;
RCU_TRACE_ME(rcupreempt_trace_try_flip_a1); RCU_TRACE_ME(rcupreempt_trace_try_flip_a1);
for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
if (rcu_try_flip_waitack_needed(cpu) && if (rcu_try_flip_waitack_needed(cpu) &&
per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) { per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) {
RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1); RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1);
@ -808,7 +809,7 @@ rcu_try_flip_waitzero(void)
/* Check to see if the sum of the "last" counters is zero. */ /* Check to see if the sum of the "last" counters is zero. */
RCU_TRACE_ME(rcupreempt_trace_try_flip_z1); RCU_TRACE_ME(rcupreempt_trace_try_flip_z1);
for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx]; sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx];
if (sum != 0) { if (sum != 0) {
RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1); RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1);
@ -823,7 +824,7 @@ rcu_try_flip_waitzero(void)
smp_mb(); /* ^^^^^^^^^^^^ */ smp_mb(); /* ^^^^^^^^^^^^ */
/* Call for a memory barrier from each CPU. */ /* Call for a memory barrier from each CPU. */
for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) { for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {
per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed; per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed;
dyntick_save_progress_counter(cpu); dyntick_save_progress_counter(cpu);
} }
@ -843,7 +844,7 @@ rcu_try_flip_waitmb(void)
int cpu; int cpu;
RCU_TRACE_ME(rcupreempt_trace_try_flip_m1); RCU_TRACE_ME(rcupreempt_trace_try_flip_m1);
for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
if (rcu_try_flip_waitmb_needed(cpu) && if (rcu_try_flip_waitmb_needed(cpu) &&
per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) { per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) {
RCU_TRACE_ME(rcupreempt_trace_try_flip_me1); RCU_TRACE_ME(rcupreempt_trace_try_flip_me1);
@ -1032,7 +1033,7 @@ void rcu_offline_cpu(int cpu)
RCU_DATA_CPU(cpu)->rcu_flipctr[0] = 0; RCU_DATA_CPU(cpu)->rcu_flipctr[0] = 0;
RCU_DATA_CPU(cpu)->rcu_flipctr[1] = 0; RCU_DATA_CPU(cpu)->rcu_flipctr[1] = 0;
cpu_clear(cpu, rcu_cpu_online_map); cpumask_clear_cpu(cpu, to_cpumask(rcu_cpu_online_map));
spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
@ -1072,7 +1073,7 @@ void __cpuinit rcu_online_cpu(int cpu)
struct rcu_data *rdp; struct rcu_data *rdp;
spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags); spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
cpu_set(cpu, rcu_cpu_online_map); cpumask_set_cpu(cpu, to_cpumask(rcu_cpu_online_map));
spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
/* /*
@ -1430,7 +1431,7 @@ void __init __rcu_init(void)
* We don't need protection against CPU-Hotplug here * We don't need protection against CPU-Hotplug here
* since * since
* a) If a CPU comes online while we are iterating over the * a) If a CPU comes online while we are iterating over the
* cpu_online_map below, we would only end up making a * cpu_online_mask below, we would only end up making a
* duplicate call to rcu_online_cpu() which sets the corresponding * duplicate call to rcu_online_cpu() which sets the corresponding
* CPU's mask in the rcu_cpu_online_map. * CPU's mask in the rcu_cpu_online_map.
* *

27
kernel/rcutorture.c
View file

@ -868,49 +868,52 @@ static int rcu_idle_cpu; /* Force all torture tasks off this CPU */
*/ */
static void rcu_torture_shuffle_tasks(void) static void rcu_torture_shuffle_tasks(void)
{ {
cpumask_t tmp_mask; cpumask_var_t tmp_mask;
int i; int i;
cpus_setall(tmp_mask); if (!alloc_cpumask_var(&tmp_mask, GFP_KERNEL))
BUG();
cpumask_setall(tmp_mask);
get_online_cpus(); get_online_cpus();
/* No point in shuffling if there is only one online CPU (ex: UP) */ /* No point in shuffling if there is only one online CPU (ex: UP) */
if (num_online_cpus() == 1) { if (num_online_cpus() == 1)
put_online_cpus(); goto out;
return;
}
if (rcu_idle_cpu != -1) if (rcu_idle_cpu != -1)
cpu_clear(rcu_idle_cpu, tmp_mask); cpumask_clear_cpu(rcu_idle_cpu, tmp_mask);
set_cpus_allowed_ptr(current, &tmp_mask); set_cpus_allowed_ptr(current, tmp_mask);
if (reader_tasks) { if (reader_tasks) {
for (i = 0; i < nrealreaders; i++) for (i = 0; i < nrealreaders; i++)
if (reader_tasks[i]) if (reader_tasks[i])
set_cpus_allowed_ptr(reader_tasks[i], set_cpus_allowed_ptr(reader_tasks[i],
&tmp_mask); tmp_mask);
} }
if (fakewriter_tasks) { if (fakewriter_tasks) {
for (i = 0; i < nfakewriters; i++) for (i = 0; i < nfakewriters; i++)
if (fakewriter_tasks[i]) if (fakewriter_tasks[i])
set_cpus_allowed_ptr(fakewriter_tasks[i], set_cpus_allowed_ptr(fakewriter_tasks[i],
&tmp_mask); tmp_mask);
} }
if (writer_task) if (writer_task)
set_cpus_allowed_ptr(writer_task, &tmp_mask); set_cpus_allowed_ptr(writer_task, tmp_mask);
if (stats_task) if (stats_task)
set_cpus_allowed_ptr(stats_task, &tmp_mask); set_cpus_allowed_ptr(stats_task, tmp_mask);
if (rcu_idle_cpu == -1) if (rcu_idle_cpu == -1)
rcu_idle_cpu = num_online_cpus() - 1; rcu_idle_cpu = num_online_cpus() - 1;
else else
rcu_idle_cpu--; rcu_idle_cpu--;
out:
put_online_cpus(); put_online_cpus();
free_cpumask_var(tmp_mask);
} }
/* Shuffle tasks across CPUs, with the intent of allowing each CPU in the /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the