SUNRPC: Micro-optimise when the task is known not to be sleeping
In cases where we know the task is not sleeping, try to optimise away the indirect call to task->tk_action() by replacing it with a direct call. Only change tail calls, to allow gcc to perform tail call elimination. Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>
This commit is contained in:
Trond Myklebust
Родитель
03e51d32da
Коммит
009a82f643
|
|
@ -304,4 +304,12 @@ rpc_clnt_swap_deactivate(struct rpc_clnt *clnt)
|
|||
}
|
||||
#endif /* CONFIG_SUNRPC_SWAP */
|
||||
|
||||
static inline bool
|
||||
rpc_task_need_resched(const struct rpc_task *task)
|
||||
{
|
||||
if (RPC_IS_QUEUED(task) || task->tk_callback)
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* _LINUX_SUNRPC_SCHED_H_ */
|
||||
|
|
|
|||
|
|
@ -1540,6 +1540,7 @@ call_start(struct rpc_task *task)
|
|||
clnt->cl_stats->rpccnt++;
|
||||
task->tk_action = call_reserve;
|
||||
rpc_task_set_transport(task, clnt);
|
||||
call_reserve(task);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1553,6 +1554,9 @@ call_reserve(struct rpc_task *task)
|
|||
task->tk_status = 0;
|
||||
task->tk_action = call_reserveresult;
|
||||
xprt_reserve(task);
|
||||
if (rpc_task_need_resched(task))
|
||||
return;
|
||||
call_reserveresult(task);
|
||||
}
|
||||
|
||||
static void call_retry_reserve(struct rpc_task *task);
|
||||
|
|
@ -1575,6 +1579,7 @@ call_reserveresult(struct rpc_task *task)
|
|||
if (status >= 0) {
|
||||
if (task->tk_rqstp) {
|
||||
task->tk_action = call_refresh;
|
||||
call_refresh(task);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1600,6 +1605,7 @@ call_reserveresult(struct rpc_task *task)
|
|||
/* fall through */
|
||||
case -EAGAIN: /* woken up; retry */
|
||||
task->tk_action = call_retry_reserve;
|
||||
call_retry_reserve(task);
|
||||
return;
|
||||
case -EIO: /* probably a shutdown */
|
||||
break;
|
||||
|
|
@ -1622,6 +1628,9 @@ call_retry_reserve(struct rpc_task *task)
|
|||
task->tk_status = 0;
|
||||
task->tk_action = call_reserveresult;
|
||||
xprt_retry_reserve(task);
|
||||
if (rpc_task_need_resched(task))
|
||||
return;
|
||||
call_reserveresult(task);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1636,6 +1645,9 @@ call_refresh(struct rpc_task *task)
|
|||
task->tk_status = 0;
|
||||
task->tk_client->cl_stats->rpcauthrefresh++;
|
||||
rpcauth_refreshcred(task);
|
||||
if (rpc_task_need_resched(task))
|
||||
return;
|
||||
call_refreshresult(task);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1654,6 +1666,7 @@ call_refreshresult(struct rpc_task *task)
|
|||
case 0:
|
||||
if (rpcauth_uptodatecred(task)) {
|
||||
task->tk_action = call_allocate;
|
||||
call_allocate(task);
|
||||
return;
|
||||
}
|
||||
/* Use rate-limiting and a max number of retries if refresh
|
||||
|
|
@ -1672,6 +1685,7 @@ call_refreshresult(struct rpc_task *task)
|
|||
task->tk_cred_retry--;
|
||||
dprintk("RPC: %5u %s: retry refresh creds\n",
|
||||
task->tk_pid, __func__);
|
||||
call_refresh(task);
|
||||
return;
|
||||
}
|
||||
dprintk("RPC: %5u %s: refresh creds failed with error %d\n",
|
||||
|
|
@ -1697,8 +1711,10 @@ call_allocate(struct rpc_task *task)
|
|||
task->tk_status = 0;
|
||||
task->tk_action = call_encode;
|
||||
|
||||
if (req->rq_buffer)
|
||||
if (req->rq_buffer) {
|
||||
call_encode(task);
|
||||
return;
|
||||
}
|
||||
|
||||
if (proc->p_proc != 0) {
|
||||
BUG_ON(proc->p_arglen == 0);
|
||||
|
|
@ -1719,8 +1735,12 @@ call_allocate(struct rpc_task *task)
|
|||
|
||||
status = xprt->ops->buf_alloc(task);
|
||||
xprt_inject_disconnect(xprt);
|
||||
if (status == 0)
|
||||
if (status == 0) {
|
||||
if (rpc_task_need_resched(task))
|
||||
return;
|
||||
call_encode(task);
|
||||
return;
|
||||
}
|
||||
if (status != -ENOMEM) {
|
||||
rpc_exit(task, status);
|
||||
return;
|
||||
|
|
@ -1803,12 +1823,8 @@ call_encode(struct rpc_task *task)
|
|||
xprt_request_enqueue_receive(task);
|
||||
xprt_request_enqueue_transmit(task);
|
||||
out:
|
||||
task->tk_action = call_transmit;
|
||||
/* Check that the connection is OK */
|
||||
if (!xprt_bound(task->tk_xprt))
|
||||
task->tk_action = call_bind;
|
||||
else if (!xprt_connected(task->tk_xprt))
|
||||
task->tk_action = call_connect;
|
||||
task->tk_action = call_bind;
|
||||
call_bind(task);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1842,14 +1858,17 @@ call_bind(struct rpc_task *task)
|
|||
return;
|
||||
}
|
||||
|
||||
if (xprt_bound(xprt)) {
|
||||
task->tk_action = call_connect;
|
||||
call_connect(task);
|
||||
return;
|
||||
}
|
||||
|
||||
dprint_status(task);
|
||||
|
||||
task->tk_action = call_connect;
|
||||
if (!xprt_bound(xprt)) {
|
||||
task->tk_action = call_bind_status;
|
||||
task->tk_timeout = xprt->bind_timeout;
|
||||
xprt->ops->rpcbind(task);
|
||||
}
|
||||
task->tk_action = call_bind_status;
|
||||
task->tk_timeout = xprt->bind_timeout;
|
||||
xprt->ops->rpcbind(task);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1869,6 +1888,7 @@ call_bind_status(struct rpc_task *task)
|
|||
dprint_status(task);
|
||||
task->tk_status = 0;
|
||||
task->tk_action = call_connect;
|
||||
call_connect(task);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1949,21 +1969,24 @@ call_connect(struct rpc_task *task)
|
|||
return;
|
||||
}
|
||||
|
||||
if (xprt_connected(xprt)) {
|
||||
task->tk_action = call_transmit;
|
||||
call_transmit(task);
|
||||
return;
|
||||
}
|
||||
|
||||
dprintk("RPC: %5u call_connect xprt %p %s connected\n",
|
||||
task->tk_pid, xprt,
|
||||
(xprt_connected(xprt) ? "is" : "is not"));
|
||||
|
||||
task->tk_action = call_transmit;
|
||||
if (!xprt_connected(xprt)) {
|
||||
task->tk_action = call_connect_status;
|
||||
if (task->tk_status < 0)
|
||||
return;
|
||||
if (task->tk_flags & RPC_TASK_NOCONNECT) {
|
||||
rpc_exit(task, -ENOTCONN);
|
||||
return;
|
||||
}
|
||||
xprt_connect(task);
|
||||
task->tk_action = call_connect_status;
|
||||
if (task->tk_status < 0)
|
||||
return;
|
||||
if (task->tk_flags & RPC_TASK_NOCONNECT) {
|
||||
rpc_exit(task, -ENOTCONN);
|
||||
return;
|
||||
}
|
||||
xprt_connect(task);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -2016,6 +2039,7 @@ call_connect_status(struct rpc_task *task)
|
|||
case 0:
|
||||
clnt->cl_stats->netreconn++;
|
||||
task->tk_action = call_transmit;
|
||||
call_transmit(task);
|
||||
return;
|
||||
}
|
||||
rpc_exit(task, status);
|
||||
|
|
@ -2040,19 +2064,20 @@ call_transmit(struct rpc_task *task)
|
|||
dprint_status(task);
|
||||
|
||||
task->tk_action = call_transmit_status;
|
||||
if (!xprt_prepare_transmit(task))
|
||||
return;
|
||||
task->tk_status = 0;
|
||||
if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
|
||||
if (!xprt_prepare_transmit(task))
|
||||
if (!xprt_connected(task->tk_xprt)) {
|
||||
task->tk_status = -ENOTCONN;
|
||||
return;
|
||||
task->tk_status = 0;
|
||||
if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
|
||||
if (!xprt_connected(task->tk_xprt)) {
|
||||
task->tk_status = -ENOTCONN;
|
||||
return;
|
||||
}
|
||||
xprt_transmit(task);
|
||||
}
|
||||
xprt_transmit(task);
|
||||
}
|
||||
xprt_end_transmit(task);
|
||||
if (rpc_task_need_resched(task))
|
||||
return;
|
||||
call_transmit_status(task);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -2067,8 +2092,12 @@ call_transmit_status(struct rpc_task *task)
|
|||
* Common case: success. Force the compiler to put this
|
||||
* test first.
|
||||
*/
|
||||
if (task->tk_status == 0) {
|
||||
xprt_request_wait_receive(task);
|
||||
if (rpc_task_transmitted(task)) {
|
||||
if (task->tk_status == 0)
|
||||
xprt_request_wait_receive(task);
|
||||
if (rpc_task_need_resched(task))
|
||||
return;
|
||||
call_status(task);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -2129,6 +2158,7 @@ call_bc_encode(struct rpc_task *task)
|
|||
{
|
||||
xprt_request_enqueue_transmit(task);
|
||||
task->tk_action = call_bc_transmit;
|
||||
call_bc_transmit(task);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -2219,6 +2249,7 @@ call_status(struct rpc_task *task)
|
|||
status = task->tk_status;
|
||||
if (status >= 0) {
|
||||
task->tk_action = call_decode;
|
||||
call_decode(task);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче