From 365187fbc04fd55766bf6a94e37e558505bf480a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 10 Mar 2014 10:55:52 -0700 Subject: [PATCH] rcu: Update cpu_needs_another_gp() for futures from non-NOCB CPUs In the old days, the only source of requests for future grace periods was NOCB CPUs. This has changed: CPUs routinely post requests for future grace periods in order to promote power efficiency and reduce OS jitter with minimal impact on grace-period latency. This commit therefore updates cpu_needs_another_gp() to invoke rcu_future_needs_gp() instead of rcu_nocb_needs_gp(). The latter is no longer used, so is now removed. This commit also adds tracing for the irq_work_queue() wakeup case. Signed-off-by: Paul E. McKenney Reviewed-by: Josh Triplett --- kernel/rcu/tree.c | 39 +++++++++++++++++++++++++++++---------- kernel/rcu/tree.h | 1 - kernel/rcu/tree_plugin.h | 18 ------------------ 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index b33c29a99df3..b4688993e956 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -323,6 +323,28 @@ cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp) rdp->nxttail[RCU_DONE_TAIL] != NULL; } +/* + * Return the root node of the specified rcu_state structure. + */ +static struct rcu_node *rcu_get_root(struct rcu_state *rsp) +{ + return &rsp->node[0]; +} + +/* + * Is there any need for future grace periods? + * Interrupts must be disabled. If the caller does not hold the root + * rnp_node structure's ->lock, the results are advisory only. + */ +static int rcu_future_needs_gp(struct rcu_state *rsp) +{ + struct rcu_node *rnp = rcu_get_root(rsp); + int idx = (ACCESS_ONCE(rnp->completed) + 1) & 0x1; + int *fp = &rnp->need_future_gp[idx]; + + return ACCESS_ONCE(*fp); +} + /* * Does the current CPU require a not-yet-started grace period? * The caller must have disabled interrupts to prevent races with @@ -335,7 +357,7 @@ cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) if (rcu_gp_in_progress(rsp)) return 0; /* No, a grace period is already in progress. */ - if (rcu_nocb_needs_gp(rsp)) + if (rcu_future_needs_gp(rsp)) return 1; /* Yes, a no-CBs CPU needs one. */ if (!rdp->nxttail[RCU_NEXT_TAIL]) return 0; /* No, this is a no-CBs (or offline) CPU. */ @@ -349,14 +371,6 @@ cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp) return 0; /* No grace period needed. */ } -/* - * Return the root node of the specified rcu_state structure. - */ -static struct rcu_node *rcu_get_root(struct rcu_state *rsp) -{ - return &rsp->node[0]; -} - /* * rcu_eqs_enter_common - current CPU is moving towards extended quiescent state * @@ -1672,6 +1686,8 @@ static void rsp_wakeup(struct irq_work *work) /* Wake up rcu_gp_kthread() to start the grace period. */ wake_up(&rsp->gp_wq); + trace_rcu_grace_period(rsp->name, ACCESS_ONCE(rsp->gpnum), + "Workqueuewoken"); } /* @@ -1706,8 +1722,11 @@ rcu_start_gp_advanced(struct rcu_state *rsp, struct rcu_node *rnp, * the wakeup to interrupt context. And don't bother waking * up the running kthread. */ - if (current != rsp->gp_kthread) + if (current != rsp->gp_kthread) { + trace_rcu_grace_period(rsp->name, ACCESS_ONCE(rsp->gpnum), + "Workqueuewake"); irq_work_queue(&rsp->wakeup_work); + } } /* diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h index 75dc3c39a02a..7b572c5c65e1 100644 --- a/kernel/rcu/tree.h +++ b/kernel/rcu/tree.h @@ -547,7 +547,6 @@ static void print_cpu_stall_info(struct rcu_state *rsp, int cpu); static void print_cpu_stall_info_end(void); static void zero_cpu_stall_ticks(struct rcu_data *rdp); static void increment_cpu_stall_ticks(void); -static int rcu_nocb_needs_gp(struct rcu_state *rsp); static void rcu_nocb_gp_set(struct rcu_node *rnp, int nrq); static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp); static void rcu_init_one_nocb(struct rcu_node *rnp); diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index f9c9057239a9..f60dd6ea8333 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -2067,19 +2067,6 @@ static int __init parse_rcu_nocb_poll(char *arg) } early_param("rcu_nocb_poll", parse_rcu_nocb_poll); -/* - * Do any no-CBs CPUs need another grace period? - * - * Interrupts must be disabled. If the caller does not hold the root - * rnp_node structure's ->lock, the results are advisory only. - */ -static int rcu_nocb_needs_gp(struct rcu_state *rsp) -{ - struct rcu_node *rnp = rcu_get_root(rsp); - - return rnp->need_future_gp[(ACCESS_ONCE(rnp->completed) + 1) & 0x1]; -} - /* * Wake up any no-CBs CPUs' kthreads that were waiting on the just-ended * grace period. @@ -2402,11 +2389,6 @@ static bool init_nocb_callback_list(struct rcu_data *rdp) #else /* #ifdef CONFIG_RCU_NOCB_CPU */ -static int rcu_nocb_needs_gp(struct rcu_state *rsp) -{ - return 0; -} - static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) { }