2019-04-30 21:42:43 +03:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
2017-01-17 16:03:22 +03:00
|
|
|
/*
|
|
|
|
* blk-mq scheduling framework
|
|
|
|
*
|
|
|
|
* Copyright (C) 2016 Jens Axboe
|
|
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/blk-mq.h>
|
|
|
|
|
|
|
|
#include <trace/events/block.h>
|
|
|
|
|
|
|
|
#include "blk.h"
|
|
|
|
#include "blk-mq.h"
|
2017-05-04 17:24:40 +03:00
|
|
|
#include "blk-mq-debugfs.h"
|
2017-01-17 16:03:22 +03:00
|
|
|
#include "blk-mq-sched.h"
|
|
|
|
#include "blk-mq-tag.h"
|
|
|
|
#include "blk-wbt.h"
|
|
|
|
|
|
|
|
void blk_mq_sched_free_hctx_data(struct request_queue *q,
|
|
|
|
void (*exit)(struct blk_mq_hw_ctx *))
|
|
|
|
{
|
|
|
|
struct blk_mq_hw_ctx *hctx;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
|
|
|
if (exit && hctx->sched_data)
|
|
|
|
exit(hctx);
|
|
|
|
kfree(hctx->sched_data);
|
|
|
|
hctx->sched_data = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_sched_free_hctx_data);
|
|
|
|
|
2018-11-20 04:52:34 +03:00
|
|
|
void blk_mq_sched_assign_ioc(struct request *rq)
|
2017-01-17 16:03:22 +03:00
|
|
|
{
|
2017-06-16 19:15:25 +03:00
|
|
|
struct request_queue *q = rq->q;
|
2018-11-21 05:12:46 +03:00
|
|
|
struct io_context *ioc;
|
2017-01-17 16:03:22 +03:00
|
|
|
struct io_cq *icq;
|
|
|
|
|
2018-11-21 05:12:46 +03:00
|
|
|
/*
|
|
|
|
* May not have an IO context if it's a passthrough request
|
|
|
|
*/
|
|
|
|
ioc = current->io_context;
|
|
|
|
if (!ioc)
|
|
|
|
return;
|
|
|
|
|
2018-11-15 22:17:28 +03:00
|
|
|
spin_lock_irq(&q->queue_lock);
|
2017-01-17 16:03:22 +03:00
|
|
|
icq = ioc_lookup_icq(ioc, q);
|
2018-11-15 22:17:28 +03:00
|
|
|
spin_unlock_irq(&q->queue_lock);
|
2017-01-17 16:03:22 +03:00
|
|
|
|
|
|
|
if (!icq) {
|
|
|
|
icq = ioc_create_icq(ioc, q, GFP_ATOMIC);
|
|
|
|
if (!icq)
|
|
|
|
return;
|
|
|
|
}
|
2017-06-16 19:15:20 +03:00
|
|
|
get_io_context(icq->ioc);
|
2017-06-16 19:15:25 +03:00
|
|
|
rq->elv.icq = icq;
|
2017-01-17 16:03:22 +03:00
|
|
|
}
|
|
|
|
|
2017-06-21 02:56:13 +03:00
|
|
|
/*
|
|
|
|
* Mark a hardware queue as needing a restart. For shared queues, maintain
|
|
|
|
* a count of how many hardware queues are marked for restart.
|
|
|
|
*/
|
2018-12-17 09:14:05 +03:00
|
|
|
void blk_mq_sched_mark_restart_hctx(struct blk_mq_hw_ctx *hctx)
|
2017-06-21 02:56:13 +03:00
|
|
|
{
|
|
|
|
if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
|
|
|
|
return;
|
|
|
|
|
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 14:31:48 +03:00
|
|
|
set_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
|
2017-06-21 02:56:13 +03:00
|
|
|
}
|
2018-12-17 09:14:05 +03:00
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_sched_mark_restart_hctx);
|
2017-06-21 02:56:13 +03:00
|
|
|
|
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 14:31:48 +03:00
|
|
|
void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx)
|
2017-06-21 02:56:13 +03:00
|
|
|
{
|
|
|
|
if (!test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
|
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 14:31:48 +03:00
|
|
|
return;
|
|
|
|
clear_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state);
|
2017-06-21 02:56:13 +03:00
|
|
|
|
blk-mq: remove synchronize_rcu() from blk_mq_del_queue_tag_set()
We have to remove synchronize_rcu() from blk_queue_cleanup(),
otherwise long delay can be caused during lun probe. For removing
it, we have to avoid to iterate the set->tag_list in IO path, eg,
blk_mq_sched_restart().
This patch reverts 5b79413946d (Revert "blk-mq: don't handle
TAG_SHARED in restart"). Given we have fixed enough IO hang issue,
and there isn't any reason to restart all queues in one tags any more,
see the following reasons:
1) blk-mq core can deal with shared-tags case well via blk_mq_get_driver_tag(),
which can wake up queues waiting for driver tag.
2) SCSI is a bit special because it may return BLK_STS_RESOURCE if queue,
target or host is ready, but SCSI built-in restart can cover all these well,
see scsi_end_request(), queue will be rerun after any request initiated from
this host/target is completed.
In my test on scsi_debug(8 luns), this patch may improve IOPS by 20% ~ 30%
when running I/O on these 8 luns concurrently.
Fixes: 705cda97ee3a ("blk-mq: Make it safe to use RCU to iterate over blk_mq_tag_set.tag_list")
Cc: Omar Sandoval <osandov@fb.com>
Cc: Bart Van Assche <bart.vanassche@wdc.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Martin K. Petersen <martin.petersen@oracle.com>
Cc: linux-scsi@vger.kernel.org
Reported-by: Andrew Jones <drjones@redhat.com>
Tested-by: Andrew Jones <drjones@redhat.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2018-06-25 14:31:48 +03:00
|
|
|
blk_mq_run_hw_queue(hctx, true);
|
2017-06-21 02:56:13 +03:00
|
|
|
}
|
|
|
|
|
2017-10-27 07:43:30 +03:00
|
|
|
/*
|
|
|
|
* Only SCSI implements .get_budget and .put_budget, and SCSI restarts
|
|
|
|
* its queue by itself in its completion handler, so we don't need to
|
|
|
|
* restart queue if .get_budget() returns BLK_STS_NO_RESOURCE.
|
|
|
|
*/
|
|
|
|
static void blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
|
2017-10-14 12:22:26 +03:00
|
|
|
{
|
|
|
|
struct request_queue *q = hctx->queue;
|
|
|
|
struct elevator_queue *e = q->elevator;
|
|
|
|
LIST_HEAD(rq_list);
|
|
|
|
|
|
|
|
do {
|
2017-10-14 12:22:29 +03:00
|
|
|
struct request *rq;
|
2017-10-14 12:22:26 +03:00
|
|
|
|
2018-11-02 01:41:41 +03:00
|
|
|
if (e->type->ops.has_work && !e->type->ops.has_work(hctx))
|
2017-10-14 12:22:26 +03:00
|
|
|
break;
|
2017-10-14 12:22:29 +03:00
|
|
|
|
2017-11-04 21:21:12 +03:00
|
|
|
if (!blk_mq_get_dispatch_budget(hctx))
|
2017-10-27 07:43:30 +03:00
|
|
|
break;
|
2017-10-14 12:22:29 +03:00
|
|
|
|
2018-11-02 01:41:41 +03:00
|
|
|
rq = e->type->ops.dispatch_request(hctx);
|
2017-10-14 12:22:29 +03:00
|
|
|
if (!rq) {
|
|
|
|
blk_mq_put_dispatch_budget(hctx);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Now this rq owns the budget which has to be released
|
|
|
|
* if this rq won't be queued to driver via .queue_rq()
|
|
|
|
* in blk_mq_dispatch_rq_list().
|
|
|
|
*/
|
2017-10-14 12:22:26 +03:00
|
|
|
list_add(&rq->queuelist, &rq_list);
|
2017-10-14 12:22:29 +03:00
|
|
|
} while (blk_mq_dispatch_rq_list(q, &rq_list, true));
|
2017-10-14 12:22:26 +03:00
|
|
|
}
|
|
|
|
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
|
|
|
static struct blk_mq_ctx *blk_mq_next_ctx(struct blk_mq_hw_ctx *hctx,
|
|
|
|
struct blk_mq_ctx *ctx)
|
|
|
|
{
|
2018-10-29 22:13:29 +03:00
|
|
|
unsigned short idx = ctx->index_hw[hctx->type];
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
|
|
|
|
|
|
|
if (++idx == hctx->nr_ctx)
|
|
|
|
idx = 0;
|
|
|
|
|
|
|
|
return hctx->ctxs[idx];
|
|
|
|
}
|
|
|
|
|
2017-10-27 07:43:30 +03:00
|
|
|
/*
|
|
|
|
* Only SCSI implements .get_budget and .put_budget, and SCSI restarts
|
|
|
|
* its queue by itself in its completion handler, so we don't need to
|
|
|
|
* restart queue if .get_budget() returns BLK_STS_NO_RESOURCE.
|
|
|
|
*/
|
|
|
|
static void blk_mq_do_dispatch_ctx(struct blk_mq_hw_ctx *hctx)
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
|
|
|
{
|
|
|
|
struct request_queue *q = hctx->queue;
|
|
|
|
LIST_HEAD(rq_list);
|
|
|
|
struct blk_mq_ctx *ctx = READ_ONCE(hctx->dispatch_from);
|
|
|
|
|
|
|
|
do {
|
|
|
|
struct request *rq;
|
|
|
|
|
|
|
|
if (!sbitmap_any_bit_set(&hctx->ctx_map))
|
|
|
|
break;
|
|
|
|
|
2017-11-04 21:21:12 +03:00
|
|
|
if (!blk_mq_get_dispatch_budget(hctx))
|
2017-10-27 07:43:30 +03:00
|
|
|
break;
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
|
|
|
|
|
|
|
rq = blk_mq_dequeue_from_ctx(hctx, ctx);
|
|
|
|
if (!rq) {
|
|
|
|
blk_mq_put_dispatch_budget(hctx);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Now this rq owns the budget which has to be released
|
|
|
|
* if this rq won't be queued to driver via .queue_rq()
|
|
|
|
* in blk_mq_dispatch_rq_list().
|
|
|
|
*/
|
|
|
|
list_add(&rq->queuelist, &rq_list);
|
|
|
|
|
|
|
|
/* round robin for fair dispatch */
|
|
|
|
ctx = blk_mq_next_ctx(hctx, rq->mq_ctx);
|
|
|
|
|
|
|
|
} while (blk_mq_dispatch_rq_list(q, &rq_list, true));
|
|
|
|
|
|
|
|
WRITE_ONCE(hctx->dispatch_from, ctx);
|
|
|
|
}
|
|
|
|
|
2017-10-27 07:43:30 +03:00
|
|
|
void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
|
2017-01-17 16:03:22 +03:00
|
|
|
{
|
blk-mq: use the right hctx when getting a driver tag fails
While dispatching requests, if we fail to get a driver tag, we mark the
hardware queue as waiting for a tag and put the requests on a
hctx->dispatch list to be run later when a driver tag is freed. However,
blk_mq_dispatch_rq_list() may dispatch requests from multiple hardware
queues if using a single-queue scheduler with a multiqueue device. If
blk_mq_get_driver_tag() fails, it doesn't update the hardware queue we
are processing. This means we end up using the hardware queue of the
previous request, which may or may not be the same as that of the
current request. If it isn't, the wrong hardware queue will end up
waiting for a tag, and the requests will be on the wrong dispatch list,
leading to a hang.
The fix is twofold:
1. Make sure we save which hardware queue we were trying to get a
request for in blk_mq_get_driver_tag() regardless of whether it
succeeds or not.
2. Make blk_mq_dispatch_rq_list() take a request_queue instead of a
blk_mq_hw_queue to make it clear that it must handle multiple
hardware queues, since I've already messed this up on a couple of
occasions.
This didn't appear in testing with nvme and mq-deadline because nvme has
more driver tags than the default number of scheduler tags. However,
with the blk_mq_update_nr_hw_queues() fix, it showed up with nbd.
Signed-off-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Jens Axboe <axboe@fb.com>
2017-04-07 17:56:26 +03:00
|
|
|
struct request_queue *q = hctx->queue;
|
|
|
|
struct elevator_queue *e = q->elevator;
|
2018-11-02 01:41:41 +03:00
|
|
|
const bool has_sched_dispatch = e && e->type->ops.dispatch_request;
|
2017-01-17 16:03:22 +03:00
|
|
|
LIST_HEAD(rq_list);
|
|
|
|
|
2017-06-18 23:24:27 +03:00
|
|
|
/* RCU or SRCU read lock is needed before checking quiesced flag */
|
|
|
|
if (unlikely(blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)))
|
2017-10-27 07:43:30 +03:00
|
|
|
return;
|
2017-01-17 16:03:22 +03:00
|
|
|
|
|
|
|
hctx->run++;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If we have previous entries on our dispatch list, grab them first for
|
|
|
|
* more fair dispatch.
|
|
|
|
*/
|
|
|
|
if (!list_empty_careful(&hctx->dispatch)) {
|
|
|
|
spin_lock(&hctx->lock);
|
|
|
|
if (!list_empty(&hctx->dispatch))
|
|
|
|
list_splice_init(&hctx->dispatch, &rq_list);
|
|
|
|
spin_unlock(&hctx->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Only ask the scheduler for requests, if we didn't have residual
|
|
|
|
* requests from the dispatch list. This is to avoid the case where
|
|
|
|
* we only ever dispatch a fraction of the requests available because
|
|
|
|
* of low device queue depth. Once we pull requests out of the IO
|
|
|
|
* scheduler, we can no longer merge or sort them. So it's best to
|
|
|
|
* leave them there for as long as we can. Mark the hw queue as
|
|
|
|
* needing a restart in that case.
|
2017-10-14 12:22:26 +03:00
|
|
|
*
|
|
|
|
* We want to dispatch from the scheduler if there was nothing
|
|
|
|
* on the dispatch list or we were able to dispatch from the
|
|
|
|
* dispatch list.
|
2017-01-17 16:03:22 +03:00
|
|
|
*/
|
2017-01-26 22:40:07 +03:00
|
|
|
if (!list_empty(&rq_list)) {
|
2017-02-22 21:58:30 +03:00
|
|
|
blk_mq_sched_mark_restart_hctx(hctx);
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
|
|
|
if (blk_mq_dispatch_rq_list(q, &rq_list, false)) {
|
|
|
|
if (has_sched_dispatch)
|
2017-10-27 07:43:30 +03:00
|
|
|
blk_mq_do_dispatch_sched(hctx);
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
|
|
|
else
|
2017-10-27 07:43:30 +03:00
|
|
|
blk_mq_do_dispatch_ctx(hctx);
|
blk-mq-sched: improve dispatching from sw queue
SCSI devices use host-wide tagset, and the shared driver tag space is
often quite big. However, there is also a queue depth for each lun(
.cmd_per_lun), which is often small, for example, on both lpfc and
qla2xxx, .cmd_per_lun is just 3.
So lots of requests may stay in sw queue, and we always flush all
belonging to same hw queue and dispatch them all to driver.
Unfortunately it is easy to cause queue busy because of the small
.cmd_per_lun. Once these requests are flushed out, they have to stay in
hctx->dispatch, and no bio merge can happen on these requests, and
sequential IO performance is harmed.
This patch introduces blk_mq_dequeue_from_ctx for dequeuing a request
from a sw queue, so that we can dispatch them in scheduler's way. We can
then avoid dequeueing too many requests from sw queue, since we don't
flush ->dispatch completely.
This patch improves dispatching from sw queue by using the .get_budget
and .put_budget callbacks.
Reviewed-by: Omar Sandoval <osandov@fb.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2017-10-14 12:22:30 +03:00
|
|
|
}
|
2017-10-14 12:22:26 +03:00
|
|
|
} else if (has_sched_dispatch) {
|
2017-10-27 07:43:30 +03:00
|
|
|
blk_mq_do_dispatch_sched(hctx);
|
2018-07-03 18:03:16 +03:00
|
|
|
} else if (hctx->dispatch_busy) {
|
|
|
|
/* dequeue request one by one from sw queue if queue is busy */
|
2017-10-27 07:43:30 +03:00
|
|
|
blk_mq_do_dispatch_ctx(hctx);
|
2017-10-14 12:22:26 +03:00
|
|
|
} else {
|
2017-01-26 22:40:07 +03:00
|
|
|
blk_mq_flush_busy_ctxs(hctx, &rq_list);
|
2017-10-14 12:22:29 +03:00
|
|
|
blk_mq_dispatch_rq_list(q, &rq_list, false);
|
2017-02-17 21:39:26 +03:00
|
|
|
}
|
2017-01-17 16:03:22 +03:00
|
|
|
}
|
|
|
|
|
2017-02-03 19:48:28 +03:00
|
|
|
bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
|
2019-06-06 13:29:01 +03:00
|
|
|
unsigned int nr_segs, struct request **merged_request)
|
2017-01-17 16:03:22 +03:00
|
|
|
{
|
|
|
|
struct request *rq;
|
|
|
|
|
2017-02-08 16:46:48 +03:00
|
|
|
switch (elv_merge(q, &rq, bio)) {
|
|
|
|
case ELEVATOR_BACK_MERGE:
|
2017-01-17 16:03:22 +03:00
|
|
|
if (!blk_mq_sched_allow_merge(q, rq, bio))
|
|
|
|
return false;
|
2019-06-06 13:29:01 +03:00
|
|
|
if (!bio_attempt_back_merge(rq, bio, nr_segs))
|
2017-02-08 16:46:48 +03:00
|
|
|
return false;
|
|
|
|
*merged_request = attempt_back_merge(q, rq);
|
|
|
|
if (!*merged_request)
|
|
|
|
elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
|
|
|
|
return true;
|
|
|
|
case ELEVATOR_FRONT_MERGE:
|
2017-01-17 16:03:22 +03:00
|
|
|
if (!blk_mq_sched_allow_merge(q, rq, bio))
|
|
|
|
return false;
|
2019-06-06 13:29:01 +03:00
|
|
|
if (!bio_attempt_front_merge(rq, bio, nr_segs))
|
2017-02-08 16:46:48 +03:00
|
|
|
return false;
|
|
|
|
*merged_request = attempt_front_merge(q, rq);
|
|
|
|
if (!*merged_request)
|
|
|
|
elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
|
|
|
|
return true;
|
2018-02-02 00:41:15 +03:00
|
|
|
case ELEVATOR_DISCARD_MERGE:
|
|
|
|
return bio_attempt_discard_merge(q, rq, bio);
|
2017-02-08 16:46:48 +03:00
|
|
|
default:
|
|
|
|
return false;
|
2017-01-17 16:03:22 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
|
|
|
|
|
2017-05-26 14:53:20 +03:00
|
|
|
/*
|
2018-05-30 10:26:07 +03:00
|
|
|
* Iterate list of requests and see if we can merge this bio with any
|
|
|
|
* of them.
|
2017-05-26 14:53:20 +03:00
|
|
|
*/
|
2018-05-30 10:26:07 +03:00
|
|
|
bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list,
|
2019-06-06 13:29:01 +03:00
|
|
|
struct bio *bio, unsigned int nr_segs)
|
2017-05-26 14:53:20 +03:00
|
|
|
{
|
|
|
|
struct request *rq;
|
|
|
|
int checked = 8;
|
|
|
|
|
2018-05-30 10:26:07 +03:00
|
|
|
list_for_each_entry_reverse(rq, list, queuelist) {
|
2017-05-26 14:53:20 +03:00
|
|
|
bool merged = false;
|
|
|
|
|
|
|
|
if (!checked--)
|
|
|
|
break;
|
|
|
|
|
|
|
|
if (!blk_rq_merge_ok(rq, bio))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
switch (blk_try_merge(rq, bio)) {
|
|
|
|
case ELEVATOR_BACK_MERGE:
|
|
|
|
if (blk_mq_sched_allow_merge(q, rq, bio))
|
2019-06-06 13:29:01 +03:00
|
|
|
merged = bio_attempt_back_merge(rq, bio,
|
|
|
|
nr_segs);
|
2017-05-26 14:53:20 +03:00
|
|
|
break;
|
|
|
|
case ELEVATOR_FRONT_MERGE:
|
|
|
|
if (blk_mq_sched_allow_merge(q, rq, bio))
|
2019-06-06 13:29:01 +03:00
|
|
|
merged = bio_attempt_front_merge(rq, bio,
|
|
|
|
nr_segs);
|
2017-05-26 14:53:20 +03:00
|
|
|
break;
|
|
|
|
case ELEVATOR_DISCARD_MERGE:
|
|
|
|
merged = bio_attempt_discard_merge(q, rq, bio);
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
return merged;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
2018-05-30 10:26:07 +03:00
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_bio_list_merge);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Reverse check our software queue for entries that we could potentially
|
|
|
|
* merge with. Currently includes a hand-wavy stop count of 8, to not spend
|
|
|
|
* too much time checking for merges.
|
|
|
|
*/
|
|
|
|
static bool blk_mq_attempt_merge(struct request_queue *q,
|
2018-12-17 18:44:05 +03:00
|
|
|
struct blk_mq_hw_ctx *hctx,
|
2019-06-06 13:29:01 +03:00
|
|
|
struct blk_mq_ctx *ctx, struct bio *bio,
|
|
|
|
unsigned int nr_segs)
|
2018-05-30 10:26:07 +03:00
|
|
|
{
|
2018-12-17 18:44:05 +03:00
|
|
|
enum hctx_type type = hctx->type;
|
|
|
|
|
2018-05-30 10:26:07 +03:00
|
|
|
lockdep_assert_held(&ctx->lock);
|
|
|
|
|
2019-06-06 13:29:01 +03:00
|
|
|
if (blk_mq_bio_list_merge(q, &ctx->rq_lists[type], bio, nr_segs)) {
|
2018-05-30 10:26:07 +03:00
|
|
|
ctx->rq_merged++;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
2017-05-26 14:53:20 +03:00
|
|
|
|
2019-06-06 13:29:01 +03:00
|
|
|
bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio,
|
|
|
|
unsigned int nr_segs)
|
2017-01-17 16:03:22 +03:00
|
|
|
{
|
|
|
|
struct elevator_queue *e = q->elevator;
|
2017-05-26 14:53:20 +03:00
|
|
|
struct blk_mq_ctx *ctx = blk_mq_get_ctx(q);
|
2019-01-24 13:25:32 +03:00
|
|
|
struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, bio->bi_opf, ctx);
|
2017-05-26 14:53:20 +03:00
|
|
|
bool ret = false;
|
2018-12-17 18:44:05 +03:00
|
|
|
enum hctx_type type;
|
2017-01-17 16:03:22 +03:00
|
|
|
|
2019-07-01 18:47:29 +03:00
|
|
|
if (e && e->type->ops.bio_merge)
|
2019-06-06 13:29:01 +03:00
|
|
|
return e->type->ops.bio_merge(hctx, bio, nr_segs);
|
2017-01-17 16:03:22 +03:00
|
|
|
|
2018-12-17 18:44:05 +03:00
|
|
|
type = hctx->type;
|
2018-07-02 12:35:59 +03:00
|
|
|
if ((hctx->flags & BLK_MQ_F_SHOULD_MERGE) &&
|
2018-12-17 18:44:05 +03:00
|
|
|
!list_empty_careful(&ctx->rq_lists[type])) {
|
2017-05-26 14:53:20 +03:00
|
|
|
/* default per sw-queue merge */
|
|
|
|
spin_lock(&ctx->lock);
|
2019-06-06 13:29:01 +03:00
|
|
|
ret = blk_mq_attempt_merge(q, hctx, ctx, bio, nr_segs);
|
2017-05-26 14:53:20 +03:00
|
|
|
spin_unlock(&ctx->lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
return ret;
|
2017-01-17 16:03:22 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq)
|
|
|
|
{
|
|
|
|
return rq_mergeable(rq) && elv_attempt_insert_merge(q, rq);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge);
|
|
|
|
|
|
|
|
void blk_mq_sched_request_inserted(struct request *rq)
|
|
|
|
{
|
|
|
|
trace_block_rq_insert(rq->q, rq);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blk_mq_sched_request_inserted);
|
|
|
|
|
2017-02-03 02:42:39 +03:00
|
|
|
static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx,
|
2017-11-02 18:24:36 +03:00
|
|
|
bool has_sched,
|
2017-02-03 02:42:39 +03:00
|
|
|
struct request *rq)
|
2017-01-17 16:03:22 +03:00
|
|
|
{
|
2020-02-25 04:04:32 +03:00
|
|
|
/*
|
|
|
|
* dispatch flush and passthrough rq directly
|
|
|
|
*
|
|
|
|
* passthrough request has to be added to hctx->dispatch directly.
|
|
|
|
* For some reason, device may be in one situation which can't
|
|
|
|
* handle FS request, so STS_RESOURCE is always returned and the
|
|
|
|
* FS request will be added to hctx->dispatch. However passthrough
|
|
|
|
* request may be required at that time for fixing the problem. If
|
|
|
|
* passthrough request is added to scheduler queue, there isn't any
|
|
|
|
* chance to dispatch it given we prioritize requests in hctx->dispatch.
|
|
|
|
*/
|
|
|
|
if ((rq->rq_flags & RQF_FLUSH_SEQ) || blk_rq_is_passthrough(rq))
|
2017-11-02 18:24:36 +03:00
|
|
|
return true;
|
|
|
|
|
2017-11-02 18:24:38 +03:00
|
|
|
if (has_sched)
|
2017-01-17 16:03:22 +03:00
|
|
|
rq->rq_flags |= RQF_SORTED;
|
|
|
|
|
2017-11-02 18:24:36 +03:00
|
|
|
return false;
|
2017-01-17 16:03:22 +03:00
|
|
|
}
|
|
|
|
|
2017-01-27 11:00:47 +03:00
|
|
|
void blk_mq_sched_insert_request(struct request *rq, bool at_head,
|
2018-01-17 19:25:58 +03:00
|
|
|
bool run_queue, bool async)
|
2017-01-27 11:00:47 +03:00
|
|
|
{
|
|
|
|
struct request_queue *q = rq->q;
|
|
|
|
struct elevator_queue *e = q->elevator;
|
|
|
|
struct blk_mq_ctx *ctx = rq->mq_ctx;
|
2018-10-30 00:06:13 +03:00
|
|
|
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
|
2017-01-27 11:00:47 +03:00
|
|
|
|
2017-11-02 18:24:36 +03:00
|
|
|
/* flush rq in flush machinery need to be dispatched directly */
|
|
|
|
if (!(rq->rq_flags & RQF_FLUSH_SEQ) && op_is_flush(rq->cmd_flags)) {
|
2017-11-02 18:24:38 +03:00
|
|
|
blk_insert_flush(rq);
|
|
|
|
goto run;
|
2017-01-27 11:00:47 +03:00
|
|
|
}
|
|
|
|
|
2017-11-02 18:24:38 +03:00
|
|
|
WARN_ON(e && (rq->tag != -1));
|
|
|
|
|
2020-02-25 04:04:32 +03:00
|
|
|
if (blk_mq_sched_bypass_insert(hctx, !!e, rq)) {
|
|
|
|
blk_mq_request_bypass_insert(rq, at_head, false);
|
2017-02-03 02:42:39 +03:00
|
|
|
goto run;
|
2020-02-25 04:04:32 +03:00
|
|
|
}
|
2017-02-03 02:42:39 +03:00
|
|
|
|
2018-11-02 01:41:41 +03:00
|
|
|
if (e && e->type->ops.insert_requests) {
|
2017-01-27 11:00:47 +03:00
|
|
|
LIST_HEAD(list);
|
|
|
|
|
|
|
|
list_add(&rq->queuelist, &list);
|
2018-11-02 01:41:41 +03:00
|
|
|
e->type->ops.insert_requests(hctx, &list, at_head);
|
2017-01-27 11:00:47 +03:00
|
|
|
} else {
|
|
|
|
spin_lock(&ctx->lock);
|
|
|
|
__blk_mq_insert_request(hctx, rq, at_head);
|
|
|
|
spin_unlock(&ctx->lock);
|
|
|
|
}
|
|
|
|
|
2017-02-03 02:42:39 +03:00
|
|
|
run:
|
2017-01-27 11:00:47 +03:00
|
|
|
if (run_queue)
|
|
|
|
blk_mq_run_hw_queue(hctx, async);
|
|
|
|
}
|
|
|
|
|
2018-10-30 20:31:51 +03:00
|
|
|
void blk_mq_sched_insert_requests(struct blk_mq_hw_ctx *hctx,
|
2017-01-27 11:00:47 +03:00
|
|
|
struct blk_mq_ctx *ctx,
|
|
|
|
struct list_head *list, bool run_queue_async)
|
|
|
|
{
|
2018-10-29 22:11:38 +03:00
|
|
|
struct elevator_queue *e;
|
blk-mq: grab .q_usage_counter when queuing request from plug code path
Just like aio/io_uring, we need to grab 2 refcount for queuing one
request, one is for submission, another is for completion.
If the request isn't queued from plug code path, the refcount grabbed
in generic_make_request() serves for submission. In theroy, this
refcount should have been released after the sumission(async run queue)
is done. blk_freeze_queue() works with blk_sync_queue() together
for avoiding race between cleanup queue and IO submission, given async
run queue activities are canceled because hctx->run_work is scheduled with
the refcount held, so it is fine to not hold the refcount when
running the run queue work function for dispatch IO.
However, if request is staggered into plug list, and finally queued
from plug code path, the refcount in submission side is actually missed.
And we may start to run queue after queue is removed because the queue's
kobject refcount isn't guaranteed to be grabbed in flushing plug list
context, then kernel oops is triggered, see the following race:
blk_mq_flush_plug_list():
blk_mq_sched_insert_requests()
insert requests to sw queue or scheduler queue
blk_mq_run_hw_queue
Because of concurrent run queue, all requests inserted above may be
completed before calling the above blk_mq_run_hw_queue. Then queue can
be freed during the above blk_mq_run_hw_queue().
Fixes the issue by grab .q_usage_counter before calling
blk_mq_sched_insert_requests() in blk_mq_flush_plug_list(). This way is
safe because the queue is absolutely alive before inserting request.
Cc: Dongli Zhang <dongli.zhang@oracle.com>
Cc: James Smart <james.smart@broadcom.com>
Cc: linux-scsi@vger.kernel.org,
Cc: Martin K . Petersen <martin.petersen@oracle.com>,
Cc: Christoph Hellwig <hch@lst.de>,
Cc: James E . J . Bottomley <jejb@linux.vnet.ibm.com>,
Reviewed-by: Bart Van Assche <bvanassche@acm.org>
Tested-by: James Smart <james.smart@broadcom.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-04-30 04:52:23 +03:00
|
|
|
struct request_queue *q = hctx->queue;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* blk_mq_sched_insert_requests() is called from flush plug
|
|
|
|
* context only, and hold one usage counter to prevent queue
|
|
|
|
* from being released.
|
|
|
|
*/
|
|
|
|
percpu_ref_get(&q->q_usage_counter);
|
2017-01-27 11:00:47 +03:00
|
|
|
|
2018-10-29 22:11:38 +03:00
|
|
|
e = hctx->queue->elevator;
|
2018-11-02 01:41:41 +03:00
|
|
|
if (e && e->type->ops.insert_requests)
|
|
|
|
e->type->ops.insert_requests(hctx, list, false);
|
2018-07-10 04:03:31 +03:00
|
|
|
else {
|
|
|
|
/*
|
|
|
|
* try to issue requests directly if the hw queue isn't
|
|
|
|
* busy in case of 'none' scheduler, and this way may save
|
|
|
|
* us one extra enqueue & dequeue to sw queue.
|
|
|
|
*/
|
2019-04-04 20:08:43 +03:00
|
|
|
if (!hctx->dispatch_busy && !e && !run_queue_async) {
|
2018-07-10 04:03:31 +03:00
|
|
|
blk_mq_try_issue_list_directly(hctx, list);
|
2019-04-04 20:08:43 +03:00
|
|
|
if (list_empty(list))
|
blk-mq: grab .q_usage_counter when queuing request from plug code path
Just like aio/io_uring, we need to grab 2 refcount for queuing one
request, one is for submission, another is for completion.
If the request isn't queued from plug code path, the refcount grabbed
in generic_make_request() serves for submission. In theroy, this
refcount should have been released after the sumission(async run queue)
is done. blk_freeze_queue() works with blk_sync_queue() together
for avoiding race between cleanup queue and IO submission, given async
run queue activities are canceled because hctx->run_work is scheduled with
the refcount held, so it is fine to not hold the refcount when
running the run queue work function for dispatch IO.
However, if request is staggered into plug list, and finally queued
from plug code path, the refcount in submission side is actually missed.
And we may start to run queue after queue is removed because the queue's
kobject refcount isn't guaranteed to be grabbed in flushing plug list
context, then kernel oops is triggered, see the following race:
blk_mq_flush_plug_list():
blk_mq_sched_insert_requests()
insert requests to sw queue or scheduler queue
blk_mq_run_hw_queue
Because of concurrent run queue, all requests inserted above may be
completed before calling the above blk_mq_run_hw_queue. Then queue can
be freed during the above blk_mq_run_hw_queue().
Fixes the issue by grab .q_usage_counter before calling
blk_mq_sched_insert_requests() in blk_mq_flush_plug_list(). This way is
safe because the queue is absolutely alive before inserting request.
Cc: Dongli Zhang <dongli.zhang@oracle.com>
Cc: James Smart <james.smart@broadcom.com>
Cc: linux-scsi@vger.kernel.org,
Cc: Martin K . Petersen <martin.petersen@oracle.com>,
Cc: Christoph Hellwig <hch@lst.de>,
Cc: James E . J . Bottomley <jejb@linux.vnet.ibm.com>,
Reviewed-by: Bart Van Assche <bvanassche@acm.org>
Tested-by: James Smart <james.smart@broadcom.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-04-30 04:52:23 +03:00
|
|
|
goto out;
|
2019-04-04 20:08:43 +03:00
|
|
|
}
|
|
|
|
blk_mq_insert_requests(hctx, ctx, list);
|
2018-07-10 04:03:31 +03:00
|
|
|
}
|
2017-01-27 11:00:47 +03:00
|
|
|
|
|
|
|
blk_mq_run_hw_queue(hctx, run_queue_async);
|
blk-mq: grab .q_usage_counter when queuing request from plug code path
Just like aio/io_uring, we need to grab 2 refcount for queuing one
request, one is for submission, another is for completion.
If the request isn't queued from plug code path, the refcount grabbed
in generic_make_request() serves for submission. In theroy, this
refcount should have been released after the sumission(async run queue)
is done. blk_freeze_queue() works with blk_sync_queue() together
for avoiding race between cleanup queue and IO submission, given async
run queue activities are canceled because hctx->run_work is scheduled with
the refcount held, so it is fine to not hold the refcount when
running the run queue work function for dispatch IO.
However, if request is staggered into plug list, and finally queued
from plug code path, the refcount in submission side is actually missed.
And we may start to run queue after queue is removed because the queue's
kobject refcount isn't guaranteed to be grabbed in flushing plug list
context, then kernel oops is triggered, see the following race:
blk_mq_flush_plug_list():
blk_mq_sched_insert_requests()
insert requests to sw queue or scheduler queue
blk_mq_run_hw_queue
Because of concurrent run queue, all requests inserted above may be
completed before calling the above blk_mq_run_hw_queue. Then queue can
be freed during the above blk_mq_run_hw_queue().
Fixes the issue by grab .q_usage_counter before calling
blk_mq_sched_insert_requests() in blk_mq_flush_plug_list(). This way is
safe because the queue is absolutely alive before inserting request.
Cc: Dongli Zhang <dongli.zhang@oracle.com>
Cc: James Smart <james.smart@broadcom.com>
Cc: linux-scsi@vger.kernel.org,
Cc: Martin K . Petersen <martin.petersen@oracle.com>,
Cc: Christoph Hellwig <hch@lst.de>,
Cc: James E . J . Bottomley <jejb@linux.vnet.ibm.com>,
Reviewed-by: Bart Van Assche <bvanassche@acm.org>
Tested-by: James Smart <james.smart@broadcom.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-04-30 04:52:23 +03:00
|
|
|
out:
|
|
|
|
percpu_ref_put(&q->q_usage_counter);
|
2017-01-27 11:00:47 +03:00
|
|
|
}
|
|
|
|
|
2017-01-17 16:03:22 +03:00
|
|
|
static void blk_mq_sched_free_tags(struct blk_mq_tag_set *set,
|
|
|
|
struct blk_mq_hw_ctx *hctx,
|
|
|
|
unsigned int hctx_idx)
|
|
|
|
{
|
|
|
|
if (hctx->sched_tags) {
|
|
|
|
blk_mq_free_rqs(set, hctx->sched_tags, hctx_idx);
|
|
|
|
blk_mq_free_rq_map(hctx->sched_tags);
|
|
|
|
hctx->sched_tags = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-05 22:01:30 +03:00
|
|
|
static int blk_mq_sched_alloc_tags(struct request_queue *q,
|
|
|
|
struct blk_mq_hw_ctx *hctx,
|
|
|
|
unsigned int hctx_idx)
|
|
|
|
{
|
|
|
|
struct blk_mq_tag_set *set = q->tag_set;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
hctx->sched_tags = blk_mq_alloc_rq_map(set, hctx_idx, q->nr_requests,
|
|
|
|
set->reserved_tags);
|
|
|
|
if (!hctx->sched_tags)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
ret = blk_mq_alloc_rqs(set, hctx->sched_tags, hctx_idx, q->nr_requests);
|
|
|
|
if (ret)
|
|
|
|
blk_mq_sched_free_tags(set, hctx, hctx_idx);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
block: free sched's request pool in blk_cleanup_queue
In theory, IO scheduler belongs to request queue, and the request pool
of sched tags belongs to the request queue too.
However, the current tags allocation interfaces are re-used for both
driver tags and sched tags, and driver tags is definitely host wide,
and doesn't belong to any request queue, same with its request pool.
So we need tagset instance for freeing request of sched tags.
Meantime, blk_mq_free_tag_set() often follows blk_cleanup_queue() in case
of non-BLK_MQ_F_TAG_SHARED, this way requires that request pool of sched
tags to be freed before calling blk_mq_free_tag_set().
Commit 47cdee29ef9d94e ("block: move blk_exit_queue into __blk_release_queue")
moves blk_exit_queue into __blk_release_queue for simplying the fast
path in generic_make_request(), then causes oops during freeing requests
of sched tags in __blk_release_queue().
Fix the above issue by move freeing request pool of sched tags into
blk_cleanup_queue(), this way is safe becasue queue has been frozen and no any
in-queue requests at that time. Freeing sched tags has to be kept in queue's
release handler becasue there might be un-completed dispatch activity
which might refer to sched tags.
Cc: Bart Van Assche <bvanassche@acm.org>
Cc: Christoph Hellwig <hch@lst.de>
Fixes: 47cdee29ef9d94e485eb08f962c74943023a5271 ("block: move blk_exit_queue into __blk_release_queue")
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-06-04 16:08:02 +03:00
|
|
|
/* called in queue's release handler, tagset has gone away */
|
2017-04-07 17:52:27 +03:00
|
|
|
static void blk_mq_sched_tags_teardown(struct request_queue *q)
|
2017-01-17 16:03:22 +03:00
|
|
|
{
|
|
|
|
struct blk_mq_hw_ctx *hctx;
|
2017-04-05 22:01:30 +03:00
|
|
|
int i;
|
|
|
|
|
block: free sched's request pool in blk_cleanup_queue
In theory, IO scheduler belongs to request queue, and the request pool
of sched tags belongs to the request queue too.
However, the current tags allocation interfaces are re-used for both
driver tags and sched tags, and driver tags is definitely host wide,
and doesn't belong to any request queue, same with its request pool.
So we need tagset instance for freeing request of sched tags.
Meantime, blk_mq_free_tag_set() often follows blk_cleanup_queue() in case
of non-BLK_MQ_F_TAG_SHARED, this way requires that request pool of sched
tags to be freed before calling blk_mq_free_tag_set().
Commit 47cdee29ef9d94e ("block: move blk_exit_queue into __blk_release_queue")
moves blk_exit_queue into __blk_release_queue for simplying the fast
path in generic_make_request(), then causes oops during freeing requests
of sched tags in __blk_release_queue().
Fix the above issue by move freeing request pool of sched tags into
blk_cleanup_queue(), this way is safe becasue queue has been frozen and no any
in-queue requests at that time. Freeing sched tags has to be kept in queue's
release handler becasue there might be un-completed dispatch activity
which might refer to sched tags.
Cc: Bart Van Assche <bvanassche@acm.org>
Cc: Christoph Hellwig <hch@lst.de>
Fixes: 47cdee29ef9d94e485eb08f962c74943023a5271 ("block: move blk_exit_queue into __blk_release_queue")
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-06-04 16:08:02 +03:00
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
|
|
|
if (hctx->sched_tags) {
|
|
|
|
blk_mq_free_rq_map(hctx->sched_tags);
|
|
|
|
hctx->sched_tags = NULL;
|
|
|
|
}
|
|
|
|
}
|
2017-04-05 22:01:30 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e)
|
|
|
|
{
|
|
|
|
struct blk_mq_hw_ctx *hctx;
|
2017-04-05 22:01:34 +03:00
|
|
|
struct elevator_queue *eq;
|
2017-04-05 22:01:30 +03:00
|
|
|
unsigned int i;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!e) {
|
|
|
|
q->elevator = NULL;
|
2018-06-02 10:18:09 +03:00
|
|
|
q->nr_requests = q->tag_set->queue_depth;
|
2017-04-05 22:01:30 +03:00
|
|
|
return 0;
|
|
|
|
}
|
2017-01-17 16:03:22 +03:00
|
|
|
|
|
|
|
/*
|
2017-07-03 15:37:14 +03:00
|
|
|
* Default to double of smaller one between hw queue_depth and 128,
|
|
|
|
* since we don't split into sync/async like the old code did.
|
|
|
|
* Additionally, this is a per-hw queue depth.
|
2017-01-17 16:03:22 +03:00
|
|
|
*/
|
2017-07-03 15:37:14 +03:00
|
|
|
q->nr_requests = 2 * min_t(unsigned int, q->tag_set->queue_depth,
|
|
|
|
BLKDEV_MAX_RQ);
|
2017-01-17 16:03:22 +03:00
|
|
|
|
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
2017-04-05 22:01:30 +03:00
|
|
|
ret = blk_mq_sched_alloc_tags(q, hctx, i);
|
2017-01-17 16:03:22 +03:00
|
|
|
if (ret)
|
2017-04-05 22:01:30 +03:00
|
|
|
goto err;
|
2017-01-17 16:03:22 +03:00
|
|
|
}
|
|
|
|
|
2018-11-02 01:41:41 +03:00
|
|
|
ret = e->ops.init_sched(q, e);
|
2017-04-05 22:01:30 +03:00
|
|
|
if (ret)
|
|
|
|
goto err;
|
2017-01-17 16:03:22 +03:00
|
|
|
|
2017-05-04 17:24:40 +03:00
|
|
|
blk_mq_debugfs_register_sched(q);
|
|
|
|
|
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
2018-11-02 01:41:41 +03:00
|
|
|
if (e->ops.init_hctx) {
|
|
|
|
ret = e->ops.init_hctx(hctx, i);
|
2017-04-05 22:01:34 +03:00
|
|
|
if (ret) {
|
|
|
|
eq = q->elevator;
|
block: free sched's request pool in blk_cleanup_queue
In theory, IO scheduler belongs to request queue, and the request pool
of sched tags belongs to the request queue too.
However, the current tags allocation interfaces are re-used for both
driver tags and sched tags, and driver tags is definitely host wide,
and doesn't belong to any request queue, same with its request pool.
So we need tagset instance for freeing request of sched tags.
Meantime, blk_mq_free_tag_set() often follows blk_cleanup_queue() in case
of non-BLK_MQ_F_TAG_SHARED, this way requires that request pool of sched
tags to be freed before calling blk_mq_free_tag_set().
Commit 47cdee29ef9d94e ("block: move blk_exit_queue into __blk_release_queue")
moves blk_exit_queue into __blk_release_queue for simplying the fast
path in generic_make_request(), then causes oops during freeing requests
of sched tags in __blk_release_queue().
Fix the above issue by move freeing request pool of sched tags into
blk_cleanup_queue(), this way is safe becasue queue has been frozen and no any
in-queue requests at that time. Freeing sched tags has to be kept in queue's
release handler becasue there might be un-completed dispatch activity
which might refer to sched tags.
Cc: Bart Van Assche <bvanassche@acm.org>
Cc: Christoph Hellwig <hch@lst.de>
Fixes: 47cdee29ef9d94e485eb08f962c74943023a5271 ("block: move blk_exit_queue into __blk_release_queue")
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-06-04 16:08:02 +03:00
|
|
|
blk_mq_sched_free_requests(q);
|
2017-04-05 22:01:34 +03:00
|
|
|
blk_mq_exit_sched(q, eq);
|
|
|
|
kobject_put(&eq->kobj);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
}
|
2017-05-04 17:24:40 +03:00
|
|
|
blk_mq_debugfs_register_sched_hctx(q, hctx);
|
2017-04-05 22:01:34 +03:00
|
|
|
}
|
|
|
|
|
2017-01-17 16:03:22 +03:00
|
|
|
return 0;
|
|
|
|
|
2017-04-05 22:01:30 +03:00
|
|
|
err:
|
block: free sched's request pool in blk_cleanup_queue
In theory, IO scheduler belongs to request queue, and the request pool
of sched tags belongs to the request queue too.
However, the current tags allocation interfaces are re-used for both
driver tags and sched tags, and driver tags is definitely host wide,
and doesn't belong to any request queue, same with its request pool.
So we need tagset instance for freeing request of sched tags.
Meantime, blk_mq_free_tag_set() often follows blk_cleanup_queue() in case
of non-BLK_MQ_F_TAG_SHARED, this way requires that request pool of sched
tags to be freed before calling blk_mq_free_tag_set().
Commit 47cdee29ef9d94e ("block: move blk_exit_queue into __blk_release_queue")
moves blk_exit_queue into __blk_release_queue for simplying the fast
path in generic_make_request(), then causes oops during freeing requests
of sched tags in __blk_release_queue().
Fix the above issue by move freeing request pool of sched tags into
blk_cleanup_queue(), this way is safe becasue queue has been frozen and no any
in-queue requests at that time. Freeing sched tags has to be kept in queue's
release handler becasue there might be un-completed dispatch activity
which might refer to sched tags.
Cc: Bart Van Assche <bvanassche@acm.org>
Cc: Christoph Hellwig <hch@lst.de>
Fixes: 47cdee29ef9d94e485eb08f962c74943023a5271 ("block: move blk_exit_queue into __blk_release_queue")
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-06-04 16:08:02 +03:00
|
|
|
blk_mq_sched_free_requests(q);
|
2017-04-07 17:52:27 +03:00
|
|
|
blk_mq_sched_tags_teardown(q);
|
|
|
|
q->elevator = NULL;
|
2017-04-05 22:01:30 +03:00
|
|
|
return ret;
|
2017-01-17 16:03:22 +03:00
|
|
|
}
|
2017-01-14 00:43:58 +03:00
|
|
|
|
block: free sched's request pool in blk_cleanup_queue
In theory, IO scheduler belongs to request queue, and the request pool
of sched tags belongs to the request queue too.
However, the current tags allocation interfaces are re-used for both
driver tags and sched tags, and driver tags is definitely host wide,
and doesn't belong to any request queue, same with its request pool.
So we need tagset instance for freeing request of sched tags.
Meantime, blk_mq_free_tag_set() often follows blk_cleanup_queue() in case
of non-BLK_MQ_F_TAG_SHARED, this way requires that request pool of sched
tags to be freed before calling blk_mq_free_tag_set().
Commit 47cdee29ef9d94e ("block: move blk_exit_queue into __blk_release_queue")
moves blk_exit_queue into __blk_release_queue for simplying the fast
path in generic_make_request(), then causes oops during freeing requests
of sched tags in __blk_release_queue().
Fix the above issue by move freeing request pool of sched tags into
blk_cleanup_queue(), this way is safe becasue queue has been frozen and no any
in-queue requests at that time. Freeing sched tags has to be kept in queue's
release handler becasue there might be un-completed dispatch activity
which might refer to sched tags.
Cc: Bart Van Assche <bvanassche@acm.org>
Cc: Christoph Hellwig <hch@lst.de>
Fixes: 47cdee29ef9d94e485eb08f962c74943023a5271 ("block: move blk_exit_queue into __blk_release_queue")
Tested-by: Yi Zhang <yi.zhang@redhat.com>
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
2019-06-04 16:08:02 +03:00
|
|
|
/*
|
|
|
|
* called in either blk_queue_cleanup or elevator_switch, tagset
|
|
|
|
* is required for freeing requests
|
|
|
|
*/
|
|
|
|
void blk_mq_sched_free_requests(struct request_queue *q)
|
|
|
|
{
|
|
|
|
struct blk_mq_hw_ctx *hctx;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
|
|
|
if (hctx->sched_tags)
|
|
|
|
blk_mq_free_rqs(q->tag_set, hctx->sched_tags, i);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-07 17:52:27 +03:00
|
|
|
void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e)
|
|
|
|
{
|
2017-04-05 22:01:34 +03:00
|
|
|
struct blk_mq_hw_ctx *hctx;
|
|
|
|
unsigned int i;
|
|
|
|
|
2017-05-04 17:24:40 +03:00
|
|
|
queue_for_each_hw_ctx(q, hctx, i) {
|
|
|
|
blk_mq_debugfs_unregister_sched_hctx(hctx);
|
2018-11-02 01:41:41 +03:00
|
|
|
if (e->type->ops.exit_hctx && hctx->sched_data) {
|
|
|
|
e->type->ops.exit_hctx(hctx, i);
|
2017-05-04 17:24:40 +03:00
|
|
|
hctx->sched_data = NULL;
|
2017-04-05 22:01:34 +03:00
|
|
|
}
|
|
|
|
}
|
2017-05-04 17:24:40 +03:00
|
|
|
blk_mq_debugfs_unregister_sched(q);
|
2018-11-02 01:41:41 +03:00
|
|
|
if (e->type->ops.exit_sched)
|
|
|
|
e->type->ops.exit_sched(e);
|
2017-04-07 17:52:27 +03:00
|
|
|
blk_mq_sched_tags_teardown(q);
|
|
|
|
q->elevator = NULL;
|
|
|
|
}
|