2009-12-03 20:59:42 +03:00
|
|
|
/*
|
|
|
|
* Common Block IO controller cgroup interface
|
|
|
|
*
|
|
|
|
* Based on ideas and code from CFQ, CFS and BFQ:
|
|
|
|
* Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
|
|
|
|
*
|
|
|
|
* Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
|
|
|
|
* Paolo Valente <paolo.valente@unimore.it>
|
|
|
|
*
|
|
|
|
* Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
|
|
|
|
* Nauman Rafique <nauman@google.com>
|
|
|
|
*/
|
|
|
|
#include <linux/ioprio.h>
|
2009-12-03 20:59:49 +03:00
|
|
|
#include <linux/seq_file.h>
|
|
|
|
#include <linux/kdev_t.h>
|
2009-12-04 18:36:41 +03:00
|
|
|
#include <linux/module.h>
|
2009-12-07 11:29:39 +03:00
|
|
|
#include <linux/err.h>
|
2010-04-02 02:01:41 +04:00
|
|
|
#include <linux/blkdev.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 11:04:11 +03:00
|
|
|
#include <linux/slab.h>
|
2010-04-13 12:05:49 +04:00
|
|
|
#include <linux/genhd.h>
|
2012-03-06 01:15:00 +04:00
|
|
|
#include <linux/delay.h>
|
2012-03-20 02:10:56 +04:00
|
|
|
#include <linux/atomic.h>
|
2012-03-06 01:15:00 +04:00
|
|
|
#include "blk-cgroup.h"
|
2012-03-06 01:15:12 +04:00
|
|
|
#include "blk.h"
|
2009-12-04 18:36:42 +03:00
|
|
|
|
2010-04-09 10:31:19 +04:00
|
|
|
#define MAX_KEY_LEN 100
|
|
|
|
|
2009-12-04 18:36:42 +03:00
|
|
|
static DEFINE_SPINLOCK(blkio_list_lock);
|
|
|
|
static LIST_HEAD(blkio_list);
|
2009-12-03 20:59:47 +03:00
|
|
|
|
2012-03-06 01:15:13 +04:00
|
|
|
static DEFINE_MUTEX(all_q_mutex);
|
|
|
|
static LIST_HEAD(all_q_list);
|
|
|
|
|
2012-03-08 22:53:56 +04:00
|
|
|
/* List of groups pending per cpu stats allocation */
|
|
|
|
static DEFINE_SPINLOCK(alloc_list_lock);
|
|
|
|
static LIST_HEAD(alloc_list);
|
|
|
|
|
|
|
|
static void blkio_stat_alloc_fn(struct work_struct *);
|
|
|
|
static DECLARE_DELAYED_WORK(blkio_stat_alloc_work, blkio_stat_alloc_fn);
|
|
|
|
|
2009-12-03 20:59:42 +03:00
|
|
|
struct blkio_cgroup blkio_root_cgroup = { .weight = 2*BLKIO_WEIGHT_DEFAULT };
|
2009-12-04 18:36:41 +03:00
|
|
|
EXPORT_SYMBOL_GPL(blkio_root_cgroup);
|
|
|
|
|
2012-03-06 01:15:04 +04:00
|
|
|
static struct blkio_policy_type *blkio_policy[BLKIO_NR_POLICIES];
|
|
|
|
|
2009-12-03 20:59:42 +03:00
|
|
|
struct blkio_cgroup *cgroup_to_blkio_cgroup(struct cgroup *cgroup)
|
|
|
|
{
|
|
|
|
return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
|
|
|
|
struct blkio_cgroup, css);
|
|
|
|
}
|
2009-12-04 18:36:41 +03:00
|
|
|
EXPORT_SYMBOL_GPL(cgroup_to_blkio_cgroup);
|
2009-12-03 20:59:42 +03:00
|
|
|
|
2012-03-06 01:15:28 +04:00
|
|
|
static struct blkio_cgroup *task_blkio_cgroup(struct task_struct *tsk)
|
2011-05-16 17:24:08 +04:00
|
|
|
{
|
|
|
|
return container_of(task_subsys_state(tsk, blkio_subsys_id),
|
|
|
|
struct blkio_cgroup, css);
|
|
|
|
}
|
2012-03-06 01:15:28 +04:00
|
|
|
|
|
|
|
struct blkio_cgroup *bio_blkio_cgroup(struct bio *bio)
|
|
|
|
{
|
|
|
|
if (bio && bio->bi_css)
|
|
|
|
return container_of(bio->bi_css, struct blkio_cgroup, css);
|
|
|
|
return task_blkio_cgroup(current);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(bio_blkio_cgroup);
|
2011-05-16 17:24:08 +04:00
|
|
|
|
2012-03-06 01:15:17 +04:00
|
|
|
static inline void blkio_update_group_weight(struct blkio_group *blkg,
|
|
|
|
int plid, unsigned int weight)
|
2010-09-16 01:06:33 +04:00
|
|
|
{
|
|
|
|
struct blkio_policy_type *blkiop;
|
|
|
|
|
|
|
|
list_for_each_entry(blkiop, &blkio_list, list) {
|
|
|
|
/* If this policy does not own the blkg, do not send updates */
|
2012-03-06 01:15:17 +04:00
|
|
|
if (blkiop->plid != plid)
|
2010-09-16 01:06:33 +04:00
|
|
|
continue;
|
|
|
|
if (blkiop->ops.blkio_update_group_weight_fn)
|
2012-03-06 01:15:03 +04:00
|
|
|
blkiop->ops.blkio_update_group_weight_fn(blkg->q,
|
2010-10-01 16:49:49 +04:00
|
|
|
blkg, weight);
|
2010-09-16 01:06:33 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-03-06 01:15:17 +04:00
|
|
|
static inline void blkio_update_group_bps(struct blkio_group *blkg, int plid,
|
2012-04-02 01:38:43 +04:00
|
|
|
u64 bps, int rw)
|
2010-09-16 01:06:34 +04:00
|
|
|
{
|
|
|
|
struct blkio_policy_type *blkiop;
|
|
|
|
|
|
|
|
list_for_each_entry(blkiop, &blkio_list, list) {
|
|
|
|
|
|
|
|
/* If this policy does not own the blkg, do not send updates */
|
2012-03-06 01:15:17 +04:00
|
|
|
if (blkiop->plid != plid)
|
2010-09-16 01:06:34 +04:00
|
|
|
continue;
|
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
if (rw == READ && blkiop->ops.blkio_update_group_read_bps_fn)
|
2012-03-06 01:15:03 +04:00
|
|
|
blkiop->ops.blkio_update_group_read_bps_fn(blkg->q,
|
2010-10-01 16:49:49 +04:00
|
|
|
blkg, bps);
|
2010-09-16 01:06:34 +04:00
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
if (rw == WRITE && blkiop->ops.blkio_update_group_write_bps_fn)
|
2012-03-06 01:15:03 +04:00
|
|
|
blkiop->ops.blkio_update_group_write_bps_fn(blkg->q,
|
2010-10-01 16:49:49 +04:00
|
|
|
blkg, bps);
|
2010-09-16 01:06:34 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
static inline void blkio_update_group_iops(struct blkio_group *blkg, int plid,
|
|
|
|
u64 iops, int rw)
|
2010-09-16 01:06:36 +04:00
|
|
|
{
|
|
|
|
struct blkio_policy_type *blkiop;
|
|
|
|
|
|
|
|
list_for_each_entry(blkiop, &blkio_list, list) {
|
|
|
|
|
|
|
|
/* If this policy does not own the blkg, do not send updates */
|
2012-03-06 01:15:17 +04:00
|
|
|
if (blkiop->plid != plid)
|
2010-09-16 01:06:36 +04:00
|
|
|
continue;
|
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
if (rw == READ && blkiop->ops.blkio_update_group_read_iops_fn)
|
2012-03-06 01:15:03 +04:00
|
|
|
blkiop->ops.blkio_update_group_read_iops_fn(blkg->q,
|
2010-10-01 16:49:49 +04:00
|
|
|
blkg, iops);
|
2010-09-16 01:06:36 +04:00
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
if (rw == WRITE && blkiop->ops.blkio_update_group_write_iops_fn)
|
2012-03-06 01:15:03 +04:00
|
|
|
blkiop->ops.blkio_update_group_write_iops_fn(blkg->q,
|
2010-10-01 16:49:49 +04:00
|
|
|
blkg,iops);
|
2010-09-16 01:06:36 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-04-09 08:15:10 +04:00
|
|
|
#ifdef CONFIG_DEBUG_BLK_CGROUP
|
2012-03-08 22:54:00 +04:00
|
|
|
/* This should be called with the queue_lock held. */
|
2010-04-09 08:15:35 +04:00
|
|
|
static void blkio_set_start_group_wait_time(struct blkio_group *blkg,
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkio_policy_type *pol,
|
|
|
|
struct blkio_group *curr_blkg)
|
2010-04-09 08:15:35 +04:00
|
|
|
{
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkg_policy_data *pd = blkg->pd[pol->plid];
|
2012-03-06 01:15:16 +04:00
|
|
|
|
|
|
|
if (blkio_blkg_waiting(&pd->stats))
|
2010-04-09 08:15:35 +04:00
|
|
|
return;
|
|
|
|
if (blkg == curr_blkg)
|
|
|
|
return;
|
2012-03-06 01:15:16 +04:00
|
|
|
pd->stats.start_group_wait_time = sched_clock();
|
|
|
|
blkio_mark_blkg_waiting(&pd->stats);
|
2010-04-09 08:15:35 +04:00
|
|
|
}
|
|
|
|
|
2012-03-08 22:54:00 +04:00
|
|
|
/* This should be called with the queue_lock held. */
|
2010-04-09 08:15:35 +04:00
|
|
|
static void blkio_update_group_wait_time(struct blkio_group_stats *stats)
|
|
|
|
{
|
|
|
|
unsigned long long now;
|
|
|
|
|
|
|
|
if (!blkio_blkg_waiting(stats))
|
|
|
|
return;
|
|
|
|
|
|
|
|
now = sched_clock();
|
|
|
|
if (time_after64(now, stats->start_group_wait_time))
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_stat_add(&stats->group_wait_time,
|
|
|
|
now - stats->start_group_wait_time);
|
2010-04-09 08:15:35 +04:00
|
|
|
blkio_clear_blkg_waiting(stats);
|
|
|
|
}
|
|
|
|
|
2012-03-08 22:54:00 +04:00
|
|
|
/* This should be called with the queue_lock held. */
|
2010-04-09 08:15:35 +04:00
|
|
|
static void blkio_end_empty_time(struct blkio_group_stats *stats)
|
|
|
|
{
|
|
|
|
unsigned long long now;
|
|
|
|
|
|
|
|
if (!blkio_blkg_empty(stats))
|
|
|
|
return;
|
|
|
|
|
|
|
|
now = sched_clock();
|
|
|
|
if (time_after64(now, stats->start_empty_time))
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_stat_add(&stats->empty_time,
|
|
|
|
now - stats->start_empty_time);
|
2010-04-09 08:15:35 +04:00
|
|
|
blkio_clear_blkg_empty(stats);
|
|
|
|
}
|
|
|
|
|
2012-03-06 01:15:17 +04:00
|
|
|
void blkiocg_update_set_idle_time_stats(struct blkio_group *blkg,
|
|
|
|
struct blkio_policy_type *pol)
|
2010-04-09 08:15:35 +04:00
|
|
|
{
|
2012-03-08 22:54:00 +04:00
|
|
|
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
|
2010-04-09 08:15:35 +04:00
|
|
|
|
2012-03-08 22:54:00 +04:00
|
|
|
lockdep_assert_held(blkg->q->queue_lock);
|
|
|
|
BUG_ON(blkio_blkg_idling(stats));
|
|
|
|
|
|
|
|
stats->start_idle_time = sched_clock();
|
|
|
|
blkio_mark_blkg_idling(stats);
|
2010-04-09 08:15:35 +04:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_update_set_idle_time_stats);
|
|
|
|
|
2012-03-06 01:15:17 +04:00
|
|
|
void blkiocg_update_idle_time_stats(struct blkio_group *blkg,
|
|
|
|
struct blkio_policy_type *pol)
|
2010-04-09 08:15:35 +04:00
|
|
|
{
|
2012-03-08 22:54:00 +04:00
|
|
|
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
|
|
|
|
|
|
|
|
lockdep_assert_held(blkg->q->queue_lock);
|
2010-04-09 08:15:35 +04:00
|
|
|
|
|
|
|
if (blkio_blkg_idling(stats)) {
|
2012-03-08 22:54:00 +04:00
|
|
|
unsigned long long now = sched_clock();
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
if (time_after64(now, stats->start_idle_time))
|
|
|
|
blkg_stat_add(&stats->idle_time,
|
|
|
|
now - stats->start_idle_time);
|
2010-04-09 08:15:35 +04:00
|
|
|
blkio_clear_blkg_idling(stats);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_update_idle_time_stats);
|
|
|
|
|
2012-03-06 01:15:17 +04:00
|
|
|
void blkiocg_update_avg_queue_size_stats(struct blkio_group *blkg,
|
|
|
|
struct blkio_policy_type *pol)
|
2010-04-09 08:15:10 +04:00
|
|
|
{
|
2012-03-08 22:54:00 +04:00
|
|
|
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
|
2010-04-09 08:15:10 +04:00
|
|
|
|
2012-03-08 22:54:00 +04:00
|
|
|
lockdep_assert_held(blkg->q->queue_lock);
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_stat_add(&stats->avg_queue_size_sum,
|
|
|
|
blkg_rwstat_sum(&stats->queued));
|
|
|
|
blkg_stat_add(&stats->avg_queue_size_samples, 1);
|
2010-04-09 08:15:35 +04:00
|
|
|
blkio_update_group_wait_time(stats);
|
2010-04-09 08:15:10 +04:00
|
|
|
}
|
2010-04-13 21:59:17 +04:00
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_update_avg_queue_size_stats);
|
|
|
|
|
2012-03-06 01:15:17 +04:00
|
|
|
void blkiocg_set_start_empty_time(struct blkio_group *blkg,
|
|
|
|
struct blkio_policy_type *pol)
|
2010-04-14 13:22:38 +04:00
|
|
|
{
|
2012-03-08 22:54:00 +04:00
|
|
|
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
|
2010-04-14 13:22:38 +04:00
|
|
|
|
2012-03-08 22:54:00 +04:00
|
|
|
lockdep_assert_held(blkg->q->queue_lock);
|
2010-04-14 13:22:38 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
if (blkg_rwstat_sum(&stats->queued))
|
2010-04-14 13:22:38 +04:00
|
|
|
return;
|
|
|
|
|
|
|
|
/*
|
2010-04-26 21:25:11 +04:00
|
|
|
* group is already marked empty. This can happen if cfqq got new
|
|
|
|
* request in parent group and moved to this group while being added
|
|
|
|
* to service tree. Just ignore the event and move on.
|
2010-04-14 13:22:38 +04:00
|
|
|
*/
|
2012-03-08 22:54:00 +04:00
|
|
|
if (blkio_blkg_empty(stats))
|
2010-04-26 21:25:11 +04:00
|
|
|
return;
|
|
|
|
|
2010-04-14 13:22:38 +04:00
|
|
|
stats->start_empty_time = sched_clock();
|
|
|
|
blkio_mark_blkg_empty(stats);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_set_start_empty_time);
|
|
|
|
|
2010-04-13 21:59:17 +04:00
|
|
|
void blkiocg_update_dequeue_stats(struct blkio_group *blkg,
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkio_policy_type *pol,
|
|
|
|
unsigned long dequeue)
|
2010-04-13 21:59:17 +04:00
|
|
|
{
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkg_policy_data *pd = blkg->pd[pol->plid];
|
2012-03-06 01:15:16 +04:00
|
|
|
|
2012-03-08 22:54:00 +04:00
|
|
|
lockdep_assert_held(blkg->q->queue_lock);
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_stat_add(&pd->stats.dequeue, dequeue);
|
2010-04-13 21:59:17 +04:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_update_dequeue_stats);
|
2010-04-09 08:15:35 +04:00
|
|
|
#else
|
|
|
|
static inline void blkio_set_start_group_wait_time(struct blkio_group *blkg,
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkio_policy_type *pol,
|
|
|
|
struct blkio_group *curr_blkg) { }
|
|
|
|
static inline void blkio_end_empty_time(struct blkio_group_stats *stats) { }
|
2010-04-09 08:15:10 +04:00
|
|
|
#endif
|
|
|
|
|
2010-04-13 21:59:17 +04:00
|
|
|
void blkiocg_update_io_add_stats(struct blkio_group *blkg,
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkio_policy_type *pol,
|
|
|
|
struct blkio_group *curr_blkg, bool direction,
|
|
|
|
bool sync)
|
2010-04-09 08:15:10 +04:00
|
|
|
{
|
2012-03-08 22:54:00 +04:00
|
|
|
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
|
2012-04-02 01:38:42 +04:00
|
|
|
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
|
2012-03-08 22:54:00 +04:00
|
|
|
|
|
|
|
lockdep_assert_held(blkg->q->queue_lock);
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_rwstat_add(&stats->queued, rw, 1);
|
2012-03-08 22:54:00 +04:00
|
|
|
blkio_end_empty_time(stats);
|
2012-03-06 01:15:17 +04:00
|
|
|
blkio_set_start_group_wait_time(blkg, pol, curr_blkg);
|
2010-04-09 08:15:10 +04:00
|
|
|
}
|
2010-04-13 21:59:17 +04:00
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_update_io_add_stats);
|
2010-04-09 08:15:10 +04:00
|
|
|
|
2010-04-13 21:59:17 +04:00
|
|
|
void blkiocg_update_io_remove_stats(struct blkio_group *blkg,
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkio_policy_type *pol,
|
|
|
|
bool direction, bool sync)
|
2010-04-09 08:15:10 +04:00
|
|
|
{
|
2012-03-08 22:54:00 +04:00
|
|
|
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
|
2012-04-02 01:38:42 +04:00
|
|
|
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
|
2012-03-08 22:54:00 +04:00
|
|
|
|
|
|
|
lockdep_assert_held(blkg->q->queue_lock);
|
2010-04-09 08:15:10 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_rwstat_add(&stats->queued, rw, -1);
|
2010-04-09 08:15:10 +04:00
|
|
|
}
|
2010-04-13 21:59:17 +04:00
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_update_io_remove_stats);
|
2010-04-09 08:15:10 +04:00
|
|
|
|
2012-03-06 01:15:17 +04:00
|
|
|
void blkiocg_update_timeslice_used(struct blkio_group *blkg,
|
|
|
|
struct blkio_policy_type *pol,
|
|
|
|
unsigned long time,
|
|
|
|
unsigned long unaccounted_time)
|
2009-12-03 20:59:49 +03:00
|
|
|
{
|
2012-03-08 22:54:00 +04:00
|
|
|
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
|
|
|
|
|
|
|
|
lockdep_assert_held(blkg->q->queue_lock);
|
2010-04-02 02:01:24 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_stat_add(&stats->time, time);
|
2011-05-19 23:38:20 +04:00
|
|
|
#ifdef CONFIG_DEBUG_BLK_CGROUP
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_stat_add(&stats->unaccounted_time, unaccounted_time);
|
2011-05-19 23:38:20 +04:00
|
|
|
#endif
|
2009-12-03 20:59:49 +03:00
|
|
|
}
|
2010-04-02 02:01:24 +04:00
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_update_timeslice_used);
|
2009-12-03 20:59:49 +03:00
|
|
|
|
2011-05-19 23:38:28 +04:00
|
|
|
/*
|
|
|
|
* should be called under rcu read lock or queue lock to make sure blkg pointer
|
|
|
|
* is valid.
|
|
|
|
*/
|
2010-04-09 10:31:19 +04:00
|
|
|
void blkiocg_update_dispatch_stats(struct blkio_group *blkg,
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkio_policy_type *pol,
|
|
|
|
uint64_t bytes, bool direction, bool sync)
|
2010-04-02 02:01:41 +04:00
|
|
|
{
|
2012-04-02 01:38:42 +04:00
|
|
|
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkg_policy_data *pd = blkg->pd[pol->plid];
|
2011-05-19 23:38:28 +04:00
|
|
|
struct blkio_group_stats_cpu *stats_cpu;
|
2011-05-19 23:38:29 +04:00
|
|
|
unsigned long flags;
|
|
|
|
|
2012-03-08 22:53:56 +04:00
|
|
|
/* If per cpu stats are not allocated yet, don't do any accounting. */
|
|
|
|
if (pd->stats_cpu == NULL)
|
|
|
|
return;
|
|
|
|
|
2011-05-19 23:38:29 +04:00
|
|
|
/*
|
|
|
|
* Disabling interrupts to provide mutual exclusion between two
|
|
|
|
* writes on same cpu. It probably is not needed for 64bit. Not
|
|
|
|
* optimizing that case yet.
|
|
|
|
*/
|
|
|
|
local_irq_save(flags);
|
2010-04-02 02:01:41 +04:00
|
|
|
|
2012-03-06 01:15:16 +04:00
|
|
|
stats_cpu = this_cpu_ptr(pd->stats_cpu);
|
2011-05-19 23:38:28 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_stat_add(&stats_cpu->sectors, bytes >> 9);
|
|
|
|
blkg_rwstat_add(&stats_cpu->serviced, rw, 1);
|
|
|
|
blkg_rwstat_add(&stats_cpu->service_bytes, rw, bytes);
|
|
|
|
|
2011-05-19 23:38:29 +04:00
|
|
|
local_irq_restore(flags);
|
2010-04-02 02:01:41 +04:00
|
|
|
}
|
2010-04-09 10:31:19 +04:00
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_update_dispatch_stats);
|
2010-04-02 02:01:41 +04:00
|
|
|
|
2010-04-09 10:31:19 +04:00
|
|
|
void blkiocg_update_completion_stats(struct blkio_group *blkg,
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkio_policy_type *pol,
|
|
|
|
uint64_t start_time,
|
|
|
|
uint64_t io_start_time, bool direction,
|
|
|
|
bool sync)
|
2010-04-02 02:01:41 +04:00
|
|
|
{
|
2012-03-08 22:54:00 +04:00
|
|
|
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
|
2010-04-02 02:01:41 +04:00
|
|
|
unsigned long long now = sched_clock();
|
2012-04-02 01:38:42 +04:00
|
|
|
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
|
2010-04-02 02:01:41 +04:00
|
|
|
|
2012-03-08 22:54:00 +04:00
|
|
|
lockdep_assert_held(blkg->q->queue_lock);
|
|
|
|
|
2010-04-09 10:31:19 +04:00
|
|
|
if (time_after64(now, io_start_time))
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_rwstat_add(&stats->service_time, rw, now - io_start_time);
|
2010-04-09 10:31:19 +04:00
|
|
|
if (time_after64(io_start_time, start_time))
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_rwstat_add(&stats->wait_time, rw,
|
|
|
|
io_start_time - start_time);
|
2010-04-02 02:01:41 +04:00
|
|
|
}
|
2010-04-09 10:31:19 +04:00
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_update_completion_stats);
|
2010-04-02 02:01:41 +04:00
|
|
|
|
2011-05-23 12:02:19 +04:00
|
|
|
/* Merged stats are per cpu. */
|
2012-03-06 01:15:17 +04:00
|
|
|
void blkiocg_update_io_merged_stats(struct blkio_group *blkg,
|
|
|
|
struct blkio_policy_type *pol,
|
|
|
|
bool direction, bool sync)
|
2010-04-09 08:14:23 +04:00
|
|
|
{
|
2012-03-08 22:54:00 +04:00
|
|
|
struct blkio_group_stats *stats = &blkg->pd[pol->plid]->stats;
|
2012-04-02 01:38:42 +04:00
|
|
|
int rw = (direction ? REQ_WRITE : 0) | (sync ? REQ_SYNC : 0);
|
2012-03-08 22:54:00 +04:00
|
|
|
|
|
|
|
lockdep_assert_held(blkg->q->queue_lock);
|
2010-04-09 08:14:23 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_rwstat_add(&stats->merged, rw, 1);
|
2010-04-09 08:14:23 +04:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blkiocg_update_io_merged_stats);
|
|
|
|
|
2012-03-08 22:53:56 +04:00
|
|
|
/*
|
|
|
|
* Worker for allocating per cpu stat for blk groups. This is scheduled on
|
|
|
|
* the system_nrt_wq once there are some groups on the alloc_list waiting
|
|
|
|
* for allocation.
|
|
|
|
*/
|
|
|
|
static void blkio_stat_alloc_fn(struct work_struct *work)
|
|
|
|
{
|
|
|
|
static void *pcpu_stats[BLKIO_NR_POLICIES];
|
|
|
|
struct delayed_work *dwork = to_delayed_work(work);
|
|
|
|
struct blkio_group *blkg;
|
|
|
|
int i;
|
|
|
|
bool empty = false;
|
|
|
|
|
|
|
|
alloc_stats:
|
|
|
|
for (i = 0; i < BLKIO_NR_POLICIES; i++) {
|
|
|
|
if (pcpu_stats[i] != NULL)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
pcpu_stats[i] = alloc_percpu(struct blkio_group_stats_cpu);
|
|
|
|
|
|
|
|
/* Allocation failed. Try again after some time. */
|
|
|
|
if (pcpu_stats[i] == NULL) {
|
|
|
|
queue_delayed_work(system_nrt_wq, dwork,
|
|
|
|
msecs_to_jiffies(10));
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_lock_irq(&blkio_list_lock);
|
|
|
|
spin_lock(&alloc_list_lock);
|
|
|
|
|
|
|
|
/* cgroup got deleted or queue exited. */
|
|
|
|
if (!list_empty(&alloc_list)) {
|
|
|
|
blkg = list_first_entry(&alloc_list, struct blkio_group,
|
|
|
|
alloc_node);
|
|
|
|
for (i = 0; i < BLKIO_NR_POLICIES; i++) {
|
|
|
|
struct blkg_policy_data *pd = blkg->pd[i];
|
|
|
|
|
|
|
|
if (blkio_policy[i] && pd && !pd->stats_cpu)
|
|
|
|
swap(pd->stats_cpu, pcpu_stats[i]);
|
|
|
|
}
|
|
|
|
|
|
|
|
list_del_init(&blkg->alloc_node);
|
|
|
|
}
|
|
|
|
|
|
|
|
empty = list_empty(&alloc_list);
|
|
|
|
|
|
|
|
spin_unlock(&alloc_list_lock);
|
|
|
|
spin_unlock_irq(&blkio_list_lock);
|
|
|
|
|
|
|
|
if (!empty)
|
|
|
|
goto alloc_stats;
|
|
|
|
}
|
|
|
|
|
2012-03-06 01:15:14 +04:00
|
|
|
/**
|
|
|
|
* blkg_free - free a blkg
|
|
|
|
* @blkg: blkg to free
|
|
|
|
*
|
|
|
|
* Free @blkg which may be partially allocated.
|
|
|
|
*/
|
|
|
|
static void blkg_free(struct blkio_group *blkg)
|
|
|
|
{
|
2012-03-06 01:15:20 +04:00
|
|
|
int i;
|
2012-03-06 01:15:16 +04:00
|
|
|
|
|
|
|
if (!blkg)
|
|
|
|
return;
|
|
|
|
|
2012-03-06 01:15:20 +04:00
|
|
|
for (i = 0; i < BLKIO_NR_POLICIES; i++) {
|
|
|
|
struct blkg_policy_data *pd = blkg->pd[i];
|
|
|
|
|
|
|
|
if (pd) {
|
|
|
|
free_percpu(pd->stats_cpu);
|
|
|
|
kfree(pd);
|
|
|
|
}
|
2012-03-06 01:15:14 +04:00
|
|
|
}
|
2012-03-06 01:15:20 +04:00
|
|
|
|
2012-03-06 01:15:16 +04:00
|
|
|
kfree(blkg);
|
2012-03-06 01:15:14 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* blkg_alloc - allocate a blkg
|
|
|
|
* @blkcg: block cgroup the new blkg is associated with
|
|
|
|
* @q: request_queue the new blkg is associated with
|
|
|
|
*
|
2012-03-06 01:15:20 +04:00
|
|
|
* Allocate a new blkg assocating @blkcg and @q.
|
2012-03-06 01:15:14 +04:00
|
|
|
*/
|
|
|
|
static struct blkio_group *blkg_alloc(struct blkio_cgroup *blkcg,
|
2012-03-06 01:15:20 +04:00
|
|
|
struct request_queue *q)
|
2012-03-06 01:15:14 +04:00
|
|
|
{
|
|
|
|
struct blkio_group *blkg;
|
2012-03-06 01:15:20 +04:00
|
|
|
int i;
|
2012-03-06 01:15:14 +04:00
|
|
|
|
|
|
|
/* alloc and init base part */
|
|
|
|
blkg = kzalloc_node(sizeof(*blkg), GFP_ATOMIC, q->node);
|
|
|
|
if (!blkg)
|
|
|
|
return NULL;
|
|
|
|
|
2012-03-06 01:15:22 +04:00
|
|
|
blkg->q = q;
|
2012-03-06 01:15:20 +04:00
|
|
|
INIT_LIST_HEAD(&blkg->q_node);
|
2012-03-08 22:53:56 +04:00
|
|
|
INIT_LIST_HEAD(&blkg->alloc_node);
|
2012-03-06 01:15:14 +04:00
|
|
|
blkg->blkcg = blkcg;
|
2012-03-06 01:15:15 +04:00
|
|
|
blkg->refcnt = 1;
|
2012-03-06 01:15:14 +04:00
|
|
|
cgroup_path(blkcg->css.cgroup, blkg->path, sizeof(blkg->path));
|
|
|
|
|
2012-03-06 01:15:20 +04:00
|
|
|
for (i = 0; i < BLKIO_NR_POLICIES; i++) {
|
|
|
|
struct blkio_policy_type *pol = blkio_policy[i];
|
|
|
|
struct blkg_policy_data *pd;
|
2012-03-06 01:15:14 +04:00
|
|
|
|
2012-03-06 01:15:20 +04:00
|
|
|
if (!pol)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
/* alloc per-policy data and attach it to blkg */
|
|
|
|
pd = kzalloc_node(sizeof(*pd) + pol->pdata_size, GFP_ATOMIC,
|
|
|
|
q->node);
|
|
|
|
if (!pd) {
|
|
|
|
blkg_free(blkg);
|
|
|
|
return NULL;
|
|
|
|
}
|
2012-03-06 01:15:16 +04:00
|
|
|
|
2012-03-06 01:15:20 +04:00
|
|
|
blkg->pd[i] = pd;
|
|
|
|
pd->blkg = blkg;
|
2012-03-06 01:15:14 +04:00
|
|
|
}
|
|
|
|
|
2012-03-06 01:15:16 +04:00
|
|
|
/* invoke per-policy init */
|
2012-03-06 01:15:20 +04:00
|
|
|
for (i = 0; i < BLKIO_NR_POLICIES; i++) {
|
|
|
|
struct blkio_policy_type *pol = blkio_policy[i];
|
|
|
|
|
|
|
|
if (pol)
|
|
|
|
pol->ops.blkio_init_group_fn(blkg);
|
|
|
|
}
|
|
|
|
|
2012-03-06 01:15:14 +04:00
|
|
|
return blkg;
|
|
|
|
}
|
|
|
|
|
2012-03-06 01:15:06 +04:00
|
|
|
struct blkio_group *blkg_lookup_create(struct blkio_cgroup *blkcg,
|
|
|
|
struct request_queue *q,
|
|
|
|
bool for_root)
|
|
|
|
__releases(q->queue_lock) __acquires(q->queue_lock)
|
2011-05-19 23:38:28 +04:00
|
|
|
{
|
2012-03-08 22:53:56 +04:00
|
|
|
struct blkio_group *blkg;
|
2011-05-19 23:38:28 +04:00
|
|
|
|
2012-03-06 01:15:06 +04:00
|
|
|
WARN_ON_ONCE(!rcu_read_lock_held());
|
|
|
|
lockdep_assert_held(q->queue_lock);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This could be the first entry point of blkcg implementation and
|
|
|
|
* we shouldn't allow anything to go through for a bypassing queue.
|
|
|
|
* The following can be removed if blkg lookup is guaranteed to
|
|
|
|
* fail on a bypassing queue.
|
|
|
|
*/
|
|
|
|
if (unlikely(blk_queue_bypass(q)) && !for_root)
|
|
|
|
return ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY);
|
|
|
|
|
2012-03-06 01:15:20 +04:00
|
|
|
blkg = blkg_lookup(blkcg, q);
|
2012-03-06 01:15:06 +04:00
|
|
|
if (blkg)
|
|
|
|
return blkg;
|
|
|
|
|
2012-03-06 01:15:11 +04:00
|
|
|
/* blkg holds a reference to blkcg */
|
2012-03-06 01:15:06 +04:00
|
|
|
if (!css_tryget(&blkcg->css))
|
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate and initialize.
|
|
|
|
*/
|
2012-03-08 22:53:56 +04:00
|
|
|
blkg = blkg_alloc(blkcg, q);
|
2012-03-06 01:15:06 +04:00
|
|
|
|
|
|
|
/* did alloc fail? */
|
2012-03-08 22:53:56 +04:00
|
|
|
if (unlikely(!blkg)) {
|
2012-03-06 01:15:06 +04:00
|
|
|
blkg = ERR_PTR(-ENOMEM);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* insert */
|
|
|
|
spin_lock(&blkcg->lock);
|
2009-12-03 20:59:42 +03:00
|
|
|
hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
|
2012-03-06 01:15:20 +04:00
|
|
|
list_add(&blkg->q_node, &q->blkg_list);
|
2012-03-06 01:15:06 +04:00
|
|
|
spin_unlock(&blkcg->lock);
|
2012-03-08 22:53:56 +04:00
|
|
|
|
|
|
|
spin_lock(&alloc_list_lock);
|
|
|
|
list_add(&blkg->alloc_node, &alloc_list);
|
|
|
|
/* Queue per cpu stat allocation from worker thread. */
|
|
|
|
queue_delayed_work(system_nrt_wq, &blkio_stat_alloc_work, 0);
|
|
|
|
spin_unlock(&alloc_list_lock);
|
2012-03-06 01:15:06 +04:00
|
|
|
out:
|
|
|
|
return blkg;
|
2009-12-03 20:59:42 +03:00
|
|
|
}
|
2012-03-06 01:15:06 +04:00
|
|
|
EXPORT_SYMBOL_GPL(blkg_lookup_create);
|
2009-12-03 20:59:42 +03:00
|
|
|
|
|
|
|
/* called under rcu_read_lock(). */
|
2012-03-06 01:15:06 +04:00
|
|
|
struct blkio_group *blkg_lookup(struct blkio_cgroup *blkcg,
|
2012-03-06 01:15:20 +04:00
|
|
|
struct request_queue *q)
|
2009-12-03 20:59:42 +03:00
|
|
|
{
|
|
|
|
struct blkio_group *blkg;
|
|
|
|
struct hlist_node *n;
|
|
|
|
|
2012-03-06 01:15:03 +04:00
|
|
|
hlist_for_each_entry_rcu(blkg, n, &blkcg->blkg_list, blkcg_node)
|
2012-03-06 01:15:20 +04:00
|
|
|
if (blkg->q == q)
|
2009-12-03 20:59:42 +03:00
|
|
|
return blkg;
|
|
|
|
return NULL;
|
|
|
|
}
|
2012-03-06 01:15:06 +04:00
|
|
|
EXPORT_SYMBOL_GPL(blkg_lookup);
|
2009-12-03 20:59:42 +03:00
|
|
|
|
2012-03-06 01:15:20 +04:00
|
|
|
static void blkg_destroy(struct blkio_group *blkg)
|
2012-03-06 01:15:19 +04:00
|
|
|
{
|
|
|
|
struct request_queue *q = blkg->q;
|
2012-03-06 01:15:21 +04:00
|
|
|
struct blkio_cgroup *blkcg = blkg->blkcg;
|
2012-03-06 01:15:19 +04:00
|
|
|
|
|
|
|
lockdep_assert_held(q->queue_lock);
|
2012-03-06 01:15:21 +04:00
|
|
|
lockdep_assert_held(&blkcg->lock);
|
2012-03-06 01:15:19 +04:00
|
|
|
|
|
|
|
/* Something wrong if we are trying to remove same group twice */
|
2012-03-06 01:15:20 +04:00
|
|
|
WARN_ON_ONCE(list_empty(&blkg->q_node));
|
2012-03-06 01:15:21 +04:00
|
|
|
WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
|
2012-03-06 01:15:20 +04:00
|
|
|
list_del_init(&blkg->q_node);
|
2012-03-06 01:15:21 +04:00
|
|
|
hlist_del_init_rcu(&blkg->blkcg_node);
|
2012-03-06 01:15:19 +04:00
|
|
|
|
2012-03-08 22:53:56 +04:00
|
|
|
spin_lock(&alloc_list_lock);
|
|
|
|
list_del_init(&blkg->alloc_node);
|
|
|
|
spin_unlock(&alloc_list_lock);
|
|
|
|
|
2012-03-06 01:15:19 +04:00
|
|
|
/*
|
|
|
|
* Put the reference taken at the time of creation so that when all
|
|
|
|
* queues are gone, group can be destroyed.
|
|
|
|
*/
|
|
|
|
blkg_put(blkg);
|
|
|
|
}
|
|
|
|
|
2012-03-06 01:15:20 +04:00
|
|
|
/*
|
|
|
|
* XXX: This updates blkg policy data in-place for root blkg, which is
|
|
|
|
* necessary across elevator switch and policy registration as root blkgs
|
|
|
|
* aren't shot down. This broken and racy implementation is temporary.
|
|
|
|
* Eventually, blkg shoot down will be replaced by proper in-place update.
|
|
|
|
*/
|
|
|
|
void update_root_blkg_pd(struct request_queue *q, enum blkio_policy_id plid)
|
|
|
|
{
|
|
|
|
struct blkio_policy_type *pol = blkio_policy[plid];
|
|
|
|
struct blkio_group *blkg = blkg_lookup(&blkio_root_cgroup, q);
|
|
|
|
struct blkg_policy_data *pd;
|
|
|
|
|
|
|
|
if (!blkg)
|
|
|
|
return;
|
|
|
|
|
|
|
|
kfree(blkg->pd[plid]);
|
|
|
|
blkg->pd[plid] = NULL;
|
|
|
|
|
|
|
|
if (!pol)
|
|
|
|
return;
|
|
|
|
|
|
|
|
pd = kzalloc(sizeof(*pd) + pol->pdata_size, GFP_KERNEL);
|
|
|
|
WARN_ON_ONCE(!pd);
|
|
|
|
|
|
|
|
pd->stats_cpu = alloc_percpu(struct blkio_group_stats_cpu);
|
|
|
|
WARN_ON_ONCE(!pd->stats_cpu);
|
|
|
|
|
|
|
|
blkg->pd[plid] = pd;
|
|
|
|
pd->blkg = blkg;
|
|
|
|
pol->ops.blkio_init_group_fn(blkg);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(update_root_blkg_pd);
|
|
|
|
|
2012-03-06 01:15:21 +04:00
|
|
|
/**
|
|
|
|
* blkg_destroy_all - destroy all blkgs associated with a request_queue
|
|
|
|
* @q: request_queue of interest
|
|
|
|
* @destroy_root: whether to destroy root blkg or not
|
|
|
|
*
|
|
|
|
* Destroy blkgs associated with @q. If @destroy_root is %true, all are
|
|
|
|
* destroyed; otherwise, root blkg is left alone.
|
|
|
|
*/
|
2012-03-06 01:15:20 +04:00
|
|
|
void blkg_destroy_all(struct request_queue *q, bool destroy_root)
|
2012-03-06 01:15:00 +04:00
|
|
|
{
|
2012-03-06 01:15:19 +04:00
|
|
|
struct blkio_group *blkg, *n;
|
2012-03-06 01:15:00 +04:00
|
|
|
|
2012-03-06 01:15:21 +04:00
|
|
|
spin_lock_irq(q->queue_lock);
|
2012-03-06 01:15:00 +04:00
|
|
|
|
2012-03-06 01:15:21 +04:00
|
|
|
list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
|
|
|
|
struct blkio_cgroup *blkcg = blkg->blkcg;
|
2012-03-06 01:15:00 +04:00
|
|
|
|
2012-03-06 01:15:21 +04:00
|
|
|
/* skip root? */
|
|
|
|
if (!destroy_root && blkg->blkcg == &blkio_root_cgroup)
|
|
|
|
continue;
|
2012-03-06 01:15:00 +04:00
|
|
|
|
2012-03-06 01:15:21 +04:00
|
|
|
spin_lock(&blkcg->lock);
|
|
|
|
blkg_destroy(blkg);
|
|
|
|
spin_unlock(&blkcg->lock);
|
2012-03-06 01:15:00 +04:00
|
|
|
}
|
2012-03-06 01:15:21 +04:00
|
|
|
|
|
|
|
spin_unlock_irq(q->queue_lock);
|
2012-03-06 01:15:00 +04:00
|
|
|
}
|
2012-03-06 01:15:19 +04:00
|
|
|
EXPORT_SYMBOL_GPL(blkg_destroy_all);
|
2012-03-06 01:15:00 +04:00
|
|
|
|
2012-03-06 01:15:15 +04:00
|
|
|
static void blkg_rcu_free(struct rcu_head *rcu_head)
|
|
|
|
{
|
|
|
|
blkg_free(container_of(rcu_head, struct blkio_group, rcu_head));
|
|
|
|
}
|
|
|
|
|
|
|
|
void __blkg_release(struct blkio_group *blkg)
|
|
|
|
{
|
|
|
|
/* release the extra blkcg reference this blkg has been holding */
|
|
|
|
css_put(&blkg->blkcg->css);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* A group is freed in rcu manner. But having an rcu lock does not
|
|
|
|
* mean that one can access all the fields of blkg and assume these
|
|
|
|
* are valid. For example, don't try to follow throtl_data and
|
|
|
|
* request queue links.
|
|
|
|
*
|
|
|
|
* Having a reference to blkg under an rcu allows acess to only
|
|
|
|
* values local to groups like group stats and group rate limits
|
|
|
|
*/
|
|
|
|
call_rcu(&blkg->rcu_head, blkg_rcu_free);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(__blkg_release);
|
|
|
|
|
2012-03-06 01:15:17 +04:00
|
|
|
static void blkio_reset_stats_cpu(struct blkio_group *blkg, int plid)
|
2011-05-19 23:38:30 +04:00
|
|
|
{
|
2012-03-06 01:15:17 +04:00
|
|
|
struct blkg_policy_data *pd = blkg->pd[plid];
|
2012-03-08 22:53:58 +04:00
|
|
|
int cpu;
|
2012-03-08 22:53:56 +04:00
|
|
|
|
|
|
|
if (pd->stats_cpu == NULL)
|
|
|
|
return;
|
2012-03-08 22:53:58 +04:00
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
struct blkio_group_stats_cpu *sc =
|
|
|
|
per_cpu_ptr(pd->stats_cpu, cpu);
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_rwstat_reset(&sc->service_bytes);
|
|
|
|
blkg_rwstat_reset(&sc->serviced);
|
|
|
|
blkg_stat_reset(&sc->sectors);
|
2011-05-19 23:38:30 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-04-02 02:01:24 +04:00
|
|
|
static int
|
2010-04-09 10:31:19 +04:00
|
|
|
blkiocg_reset_stats(struct cgroup *cgroup, struct cftype *cftype, u64 val)
|
2010-04-02 02:01:24 +04:00
|
|
|
{
|
2012-03-08 22:53:58 +04:00
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
|
2010-04-02 02:01:24 +04:00
|
|
|
struct blkio_group *blkg;
|
|
|
|
struct hlist_node *n;
|
|
|
|
|
2012-03-06 01:15:20 +04:00
|
|
|
spin_lock(&blkio_list_lock);
|
2010-04-02 02:01:24 +04:00
|
|
|
spin_lock_irq(&blkcg->lock);
|
2012-03-08 22:53:58 +04:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Note that stat reset is racy - it doesn't synchronize against
|
|
|
|
* stat updates. This is a debug feature which shouldn't exist
|
|
|
|
* anyway. If you get hit by a race, retry.
|
|
|
|
*/
|
2010-04-02 02:01:24 +04:00
|
|
|
hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
|
2012-03-06 01:15:20 +04:00
|
|
|
struct blkio_policy_type *pol;
|
2012-03-06 01:15:16 +04:00
|
|
|
|
2012-03-06 01:15:20 +04:00
|
|
|
list_for_each_entry(pol, &blkio_list, list) {
|
|
|
|
struct blkg_policy_data *pd = blkg->pd[pol->plid];
|
2012-03-08 22:53:58 +04:00
|
|
|
struct blkio_group_stats *stats = &pd->stats;
|
|
|
|
|
|
|
|
/* queued stats shouldn't be cleared */
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_rwstat_reset(&stats->merged);
|
|
|
|
blkg_rwstat_reset(&stats->service_time);
|
|
|
|
blkg_rwstat_reset(&stats->wait_time);
|
|
|
|
blkg_stat_reset(&stats->time);
|
2010-04-09 08:15:35 +04:00
|
|
|
#ifdef CONFIG_DEBUG_BLK_CGROUP
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg_stat_reset(&stats->unaccounted_time);
|
|
|
|
blkg_stat_reset(&stats->avg_queue_size_sum);
|
|
|
|
blkg_stat_reset(&stats->avg_queue_size_samples);
|
|
|
|
blkg_stat_reset(&stats->dequeue);
|
|
|
|
blkg_stat_reset(&stats->group_wait_time);
|
|
|
|
blkg_stat_reset(&stats->idle_time);
|
|
|
|
blkg_stat_reset(&stats->empty_time);
|
2010-04-09 08:15:35 +04:00
|
|
|
#endif
|
2012-03-06 01:15:20 +04:00
|
|
|
blkio_reset_stats_cpu(blkg, pol->plid);
|
|
|
|
}
|
2010-04-02 02:01:24 +04:00
|
|
|
}
|
2011-05-19 23:38:30 +04:00
|
|
|
|
2010-04-02 02:01:24 +04:00
|
|
|
spin_unlock_irq(&blkcg->lock);
|
2012-03-06 01:15:20 +04:00
|
|
|
spin_unlock(&blkio_list_lock);
|
2010-04-02 02:01:24 +04:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
static const char *blkg_dev_name(struct blkio_group *blkg)
|
2010-04-02 02:01:24 +04:00
|
|
|
{
|
2012-04-02 01:38:42 +04:00
|
|
|
/* some drivers (floppy) instantiate a queue w/o disk registered */
|
|
|
|
if (blkg->q->backing_dev_info.dev)
|
|
|
|
return dev_name(blkg->q->backing_dev_info.dev);
|
|
|
|
return NULL;
|
2010-04-02 02:01:24 +04:00
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
/**
|
|
|
|
* blkcg_print_blkgs - helper for printing per-blkg data
|
|
|
|
* @sf: seq_file to print to
|
|
|
|
* @blkcg: blkcg of interest
|
|
|
|
* @prfill: fill function to print out a blkg
|
|
|
|
* @pol: policy in question
|
|
|
|
* @data: data to be passed to @prfill
|
|
|
|
* @show_total: to print out sum of prfill return values or not
|
|
|
|
*
|
|
|
|
* This function invokes @prfill on each blkg of @blkcg if pd for the
|
|
|
|
* policy specified by @pol exists. @prfill is invoked with @sf, the
|
|
|
|
* policy data and @data. If @show_total is %true, the sum of the return
|
|
|
|
* values from @prfill is printed with "Total" label at the end.
|
|
|
|
*
|
|
|
|
* This is to be used to construct print functions for
|
|
|
|
* cftype->read_seq_string method.
|
|
|
|
*/
|
|
|
|
static void blkcg_print_blkgs(struct seq_file *sf, struct blkio_cgroup *blkcg,
|
|
|
|
u64 (*prfill)(struct seq_file *,
|
|
|
|
struct blkg_policy_data *, int),
|
|
|
|
int pol, int data, bool show_total)
|
2011-05-19 23:38:28 +04:00
|
|
|
{
|
2012-04-02 01:38:42 +04:00
|
|
|
struct blkio_group *blkg;
|
|
|
|
struct hlist_node *n;
|
|
|
|
u64 total = 0;
|
2011-05-19 23:38:28 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
spin_lock_irq(&blkcg->lock);
|
|
|
|
hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node)
|
|
|
|
if (blkg->pd[pol])
|
|
|
|
total += prfill(sf, blkg->pd[pol], data);
|
|
|
|
spin_unlock_irq(&blkcg->lock);
|
|
|
|
|
|
|
|
if (show_total)
|
|
|
|
seq_printf(sf, "Total %llu\n", (unsigned long long)total);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* __blkg_prfill_u64 - prfill helper for a single u64 value
|
|
|
|
* @sf: seq_file to print to
|
|
|
|
* @pd: policy data of interest
|
|
|
|
* @v: value to print
|
|
|
|
*
|
|
|
|
* Print @v to @sf for the device assocaited with @pd.
|
|
|
|
*/
|
|
|
|
static u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd,
|
|
|
|
u64 v)
|
|
|
|
{
|
|
|
|
const char *dname = blkg_dev_name(pd->blkg);
|
|
|
|
|
|
|
|
if (!dname)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
|
|
|
|
return v;
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
|
|
|
|
* @sf: seq_file to print to
|
|
|
|
* @pd: policy data of interest
|
|
|
|
* @rwstat: rwstat to print
|
|
|
|
*
|
|
|
|
* Print @rwstat to @sf for the device assocaited with @pd.
|
|
|
|
*/
|
|
|
|
static u64 __blkg_prfill_rwstat(struct seq_file *sf,
|
|
|
|
struct blkg_policy_data *pd,
|
|
|
|
const struct blkg_rwstat *rwstat)
|
|
|
|
{
|
|
|
|
static const char *rwstr[] = {
|
|
|
|
[BLKG_RWSTAT_READ] = "Read",
|
|
|
|
[BLKG_RWSTAT_WRITE] = "Write",
|
|
|
|
[BLKG_RWSTAT_SYNC] = "Sync",
|
|
|
|
[BLKG_RWSTAT_ASYNC] = "Async",
|
|
|
|
};
|
|
|
|
const char *dname = blkg_dev_name(pd->blkg);
|
|
|
|
u64 v;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!dname)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
for (i = 0; i < BLKG_RWSTAT_NR; i++)
|
|
|
|
seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
|
|
|
|
(unsigned long long)rwstat->cnt[i]);
|
|
|
|
|
|
|
|
v = rwstat->cnt[BLKG_RWSTAT_READ] + rwstat->cnt[BLKG_RWSTAT_WRITE];
|
|
|
|
seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
|
|
|
|
return v;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd,
|
|
|
|
int off)
|
|
|
|
{
|
|
|
|
return __blkg_prfill_u64(sf, pd,
|
|
|
|
blkg_stat_read((void *)&pd->stats + off));
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
|
|
|
|
int off)
|
|
|
|
{
|
|
|
|
struct blkg_rwstat rwstat = blkg_rwstat_read((void *)&pd->stats + off);
|
|
|
|
|
|
|
|
return __blkg_prfill_rwstat(sf, pd, &rwstat);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* print blkg_stat specified by BLKCG_STAT_PRIV() */
|
|
|
|
static int blkcg_print_stat(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
struct seq_file *sf)
|
|
|
|
{
|
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
|
|
|
|
|
|
|
blkcg_print_blkgs(sf, blkcg, blkg_prfill_stat,
|
|
|
|
BLKCG_STAT_POL(cft->private),
|
|
|
|
BLKCG_STAT_OFF(cft->private), false);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* print blkg_rwstat specified by BLKCG_STAT_PRIV() */
|
|
|
|
static int blkcg_print_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
struct seq_file *sf)
|
|
|
|
{
|
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
|
|
|
|
|
|
|
blkcg_print_blkgs(sf, blkcg, blkg_prfill_rwstat,
|
|
|
|
BLKCG_STAT_POL(cft->private),
|
|
|
|
BLKCG_STAT_OFF(cft->private), true);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static u64 blkg_prfill_cpu_stat(struct seq_file *sf,
|
|
|
|
struct blkg_policy_data *pd, int off)
|
|
|
|
{
|
|
|
|
u64 v = 0;
|
|
|
|
int cpu;
|
2012-03-08 22:53:56 +04:00
|
|
|
|
2011-05-19 23:38:28 +04:00
|
|
|
for_each_possible_cpu(cpu) {
|
2012-04-02 01:38:42 +04:00
|
|
|
struct blkio_group_stats_cpu *sc =
|
2012-04-02 01:38:42 +04:00
|
|
|
per_cpu_ptr(pd->stats_cpu, cpu);
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
v += blkg_stat_read((void *)sc + off);
|
2011-05-19 23:38:28 +04:00
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
return __blkg_prfill_u64(sf, pd, v);
|
2011-05-19 23:38:28 +04:00
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
static u64 blkg_prfill_cpu_rwstat(struct seq_file *sf,
|
|
|
|
struct blkg_policy_data *pd, int off)
|
2011-05-19 23:38:28 +04:00
|
|
|
{
|
2012-04-02 01:38:42 +04:00
|
|
|
struct blkg_rwstat rwstat = { }, tmp;
|
|
|
|
int i, cpu;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
struct blkio_group_stats_cpu *sc =
|
|
|
|
per_cpu_ptr(pd->stats_cpu, cpu);
|
2011-05-19 23:38:28 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
tmp = blkg_rwstat_read((void *)sc + off);
|
|
|
|
for (i = 0; i < BLKG_RWSTAT_NR; i++)
|
|
|
|
rwstat.cnt[i] += tmp.cnt[i];
|
2011-05-19 23:38:28 +04:00
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
return __blkg_prfill_rwstat(sf, pd, &rwstat);
|
|
|
|
}
|
2011-05-19 23:38:28 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
/* print per-cpu blkg_stat specified by BLKCG_STAT_PRIV() */
|
|
|
|
static int blkcg_print_cpu_stat(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
struct seq_file *sf)
|
|
|
|
{
|
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
|
|
|
|
|
|
|
blkcg_print_blkgs(sf, blkcg, blkg_prfill_cpu_stat,
|
|
|
|
BLKCG_STAT_POL(cft->private),
|
|
|
|
BLKCG_STAT_OFF(cft->private), false);
|
|
|
|
return 0;
|
2011-05-19 23:38:28 +04:00
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
/* print per-cpu blkg_rwstat specified by BLKCG_STAT_PRIV() */
|
|
|
|
static int blkcg_print_cpu_rwstat(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
struct seq_file *sf)
|
2010-04-02 02:01:24 +04:00
|
|
|
{
|
2012-04-02 01:38:42 +04:00
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
2012-04-02 01:38:42 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
blkcg_print_blkgs(sf, blkcg, blkg_prfill_cpu_rwstat,
|
|
|
|
BLKCG_STAT_POL(cft->private),
|
|
|
|
BLKCG_STAT_OFF(cft->private), true);
|
|
|
|
return 0;
|
|
|
|
}
|
2010-04-02 02:01:24 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
#ifdef CONFIG_DEBUG_BLK_CGROUP
|
|
|
|
static u64 blkg_prfill_avg_queue_size(struct seq_file *sf,
|
|
|
|
struct blkg_policy_data *pd, int off)
|
|
|
|
{
|
|
|
|
u64 samples = blkg_stat_read(&pd->stats.avg_queue_size_samples);
|
|
|
|
u64 v = 0;
|
2012-03-08 22:53:59 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
if (samples) {
|
|
|
|
v = blkg_stat_read(&pd->stats.avg_queue_size_sum);
|
|
|
|
do_div(v, samples);
|
2012-04-02 01:38:42 +04:00
|
|
|
}
|
2012-04-02 01:38:42 +04:00
|
|
|
__blkg_prfill_u64(sf, pd, v);
|
|
|
|
return 0;
|
|
|
|
}
|
2012-03-08 22:53:59 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
/* print avg_queue_size */
|
|
|
|
static int blkcg_print_avg_queue_size(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
struct seq_file *sf)
|
|
|
|
{
|
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
2012-03-08 22:53:59 +04:00
|
|
|
|
2012-04-02 01:38:42 +04:00
|
|
|
blkcg_print_blkgs(sf, blkcg, blkg_prfill_avg_queue_size,
|
|
|
|
BLKIO_POLICY_PROP, 0, false);
|
|
|
|
return 0;
|
2010-04-02 02:01:24 +04:00
|
|
|
}
|
2012-04-02 01:38:42 +04:00
|
|
|
#endif /* CONFIG_DEBUG_BLK_CGROUP */
|
2010-04-02 02:01:24 +04:00
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
struct blkg_conf_ctx {
|
|
|
|
struct gendisk *disk;
|
|
|
|
struct blkio_group *blkg;
|
|
|
|
u64 v;
|
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* blkg_conf_prep - parse and prepare for per-blkg config update
|
|
|
|
* @blkcg: target block cgroup
|
|
|
|
* @input: input string
|
|
|
|
* @ctx: blkg_conf_ctx to be filled
|
|
|
|
*
|
|
|
|
* Parse per-blkg config update from @input and initialize @ctx with the
|
|
|
|
* result. @ctx->blkg points to the blkg to be updated and @ctx->v the new
|
|
|
|
* value. This function returns with RCU read locked and must be paired
|
|
|
|
* with blkg_conf_finish().
|
|
|
|
*/
|
|
|
|
static int blkg_conf_prep(struct blkio_cgroup *blkcg, const char *input,
|
|
|
|
struct blkg_conf_ctx *ctx)
|
|
|
|
__acquires(rcu)
|
2010-04-13 12:05:49 +04:00
|
|
|
{
|
2012-04-02 01:38:43 +04:00
|
|
|
struct gendisk *disk;
|
|
|
|
struct blkio_group *blkg;
|
|
|
|
char *buf, *s[4], *p, *major_s, *minor_s;
|
2011-09-21 12:22:10 +04:00
|
|
|
unsigned long major, minor;
|
2011-10-19 16:31:15 +04:00
|
|
|
int i = 0, ret = -EINVAL;
|
|
|
|
int part;
|
2010-04-13 12:05:49 +04:00
|
|
|
dev_t dev;
|
2011-09-21 12:22:10 +04:00
|
|
|
u64 temp;
|
2010-04-13 12:05:49 +04:00
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
buf = kstrdup(input, GFP_KERNEL);
|
|
|
|
if (!buf)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2010-04-13 12:05:49 +04:00
|
|
|
memset(s, 0, sizeof(s));
|
|
|
|
|
|
|
|
while ((p = strsep(&buf, " ")) != NULL) {
|
|
|
|
if (!*p)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
s[i++] = p;
|
|
|
|
|
|
|
|
/* Prevent from inputing too many things */
|
|
|
|
if (i == 3)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i != 2)
|
2011-10-19 16:31:15 +04:00
|
|
|
goto out;
|
2010-04-13 12:05:49 +04:00
|
|
|
|
|
|
|
p = strsep(&s[0], ":");
|
|
|
|
if (p != NULL)
|
|
|
|
major_s = p;
|
|
|
|
else
|
2011-10-19 16:31:15 +04:00
|
|
|
goto out;
|
2010-04-13 12:05:49 +04:00
|
|
|
|
|
|
|
minor_s = s[0];
|
|
|
|
if (!minor_s)
|
2011-10-19 16:31:15 +04:00
|
|
|
goto out;
|
2010-04-13 12:05:49 +04:00
|
|
|
|
2011-10-19 16:31:15 +04:00
|
|
|
if (strict_strtoul(major_s, 10, &major))
|
|
|
|
goto out;
|
2010-04-13 12:05:49 +04:00
|
|
|
|
2011-10-19 16:31:15 +04:00
|
|
|
if (strict_strtoul(minor_s, 10, &minor))
|
|
|
|
goto out;
|
2010-04-13 12:05:49 +04:00
|
|
|
|
|
|
|
dev = MKDEV(major, minor);
|
|
|
|
|
2011-10-19 16:31:15 +04:00
|
|
|
if (strict_strtoull(s[1], 10, &temp))
|
|
|
|
goto out;
|
2010-04-13 12:05:49 +04:00
|
|
|
|
2012-03-06 01:15:07 +04:00
|
|
|
disk = get_gendisk(dev, &part);
|
2012-03-06 01:15:08 +04:00
|
|
|
if (!disk || part)
|
2012-03-06 01:15:07 +04:00
|
|
|
goto out;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
2012-03-06 01:15:08 +04:00
|
|
|
spin_lock_irq(disk->queue->queue_lock);
|
2012-04-02 01:38:42 +04:00
|
|
|
blkg = blkg_lookup_create(blkcg, disk->queue, false);
|
2012-03-06 01:15:08 +04:00
|
|
|
spin_unlock_irq(disk->queue->queue_lock);
|
2012-03-06 01:15:07 +04:00
|
|
|
|
2012-03-06 01:15:08 +04:00
|
|
|
if (IS_ERR(blkg)) {
|
|
|
|
ret = PTR_ERR(blkg);
|
2012-04-02 01:38:43 +04:00
|
|
|
rcu_read_unlock();
|
|
|
|
put_disk(disk);
|
|
|
|
/*
|
|
|
|
* If queue was bypassing, we should retry. Do so after a
|
|
|
|
* short msleep(). It isn't strictly necessary but queue
|
|
|
|
* can be bypassing for some time and it's always nice to
|
|
|
|
* avoid busy looping.
|
|
|
|
*/
|
|
|
|
if (ret == -EBUSY) {
|
|
|
|
msleep(10);
|
|
|
|
ret = restart_syscall();
|
2010-09-16 01:06:36 +04:00
|
|
|
}
|
2012-04-02 01:38:43 +04:00
|
|
|
goto out;
|
2010-09-16 01:06:33 +04:00
|
|
|
}
|
2012-04-02 01:38:43 +04:00
|
|
|
|
|
|
|
ctx->disk = disk;
|
|
|
|
ctx->blkg = blkg;
|
|
|
|
ctx->v = temp;
|
2011-10-19 16:31:15 +04:00
|
|
|
ret = 0;
|
|
|
|
out:
|
2012-04-02 01:38:43 +04:00
|
|
|
kfree(buf);
|
2011-10-19 16:31:15 +04:00
|
|
|
return ret;
|
2010-04-13 12:05:49 +04:00
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
/**
|
|
|
|
* blkg_conf_finish - finish up per-blkg config update
|
|
|
|
* @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
|
|
|
|
*
|
|
|
|
* Finish up after per-blkg config update. This function must be paired
|
|
|
|
* with blkg_conf_prep().
|
|
|
|
*/
|
|
|
|
static void blkg_conf_finish(struct blkg_conf_ctx *ctx)
|
|
|
|
__releases(rcu)
|
2010-04-13 12:05:49 +04:00
|
|
|
{
|
2012-04-02 01:38:43 +04:00
|
|
|
rcu_read_unlock();
|
|
|
|
put_disk(ctx->disk);
|
2010-04-13 12:05:49 +04:00
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
/* for propio conf */
|
|
|
|
static u64 blkg_prfill_weight_device(struct seq_file *sf,
|
|
|
|
struct blkg_policy_data *pd, int off)
|
2010-04-13 12:05:49 +04:00
|
|
|
{
|
2012-04-02 01:38:43 +04:00
|
|
|
if (!pd->conf.weight)
|
|
|
|
return 0;
|
|
|
|
return __blkg_prfill_u64(sf, pd, pd->conf.weight);
|
2010-09-16 01:06:33 +04:00
|
|
|
}
|
2010-04-13 12:05:49 +04:00
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
static int blkcg_print_weight_device(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
struct seq_file *sf)
|
2010-04-13 12:05:49 +04:00
|
|
|
{
|
2012-04-02 01:38:43 +04:00
|
|
|
blkcg_print_blkgs(sf, cgroup_to_blkio_cgroup(cgrp),
|
|
|
|
blkg_prfill_weight_device, BLKIO_POLICY_PROP, 0,
|
|
|
|
false);
|
|
|
|
return 0;
|
2010-09-16 01:06:33 +04:00
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
static int blkcg_print_weight(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
struct seq_file *sf)
|
2010-09-16 01:06:33 +04:00
|
|
|
{
|
2012-04-02 01:38:43 +04:00
|
|
|
seq_printf(sf, "%u\n", cgroup_to_blkio_cgroup(cgrp)->weight);
|
2010-09-16 01:06:33 +04:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
static int blkcg_set_weight_device(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
const char *buf)
|
|
|
|
{
|
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
|
|
|
struct blkg_policy_data *pd;
|
|
|
|
struct blkg_conf_ctx ctx;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = blkg_conf_prep(blkcg, buf, &ctx);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
ret = -EINVAL;
|
|
|
|
pd = ctx.blkg->pd[BLKIO_POLICY_PROP];
|
|
|
|
if (pd && (!ctx.v || (ctx.v >= BLKIO_WEIGHT_MIN &&
|
|
|
|
ctx.v <= BLKIO_WEIGHT_MAX))) {
|
|
|
|
pd->conf.weight = ctx.v;
|
|
|
|
blkio_update_group_weight(ctx.blkg, BLKIO_POLICY_PROP,
|
|
|
|
ctx.v ?: blkcg->weight);
|
|
|
|
ret = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
blkg_conf_finish(&ctx);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
static int blkcg_set_weight(struct cgroup *cgrp, struct cftype *cft, u64 val)
|
2010-09-16 01:06:33 +04:00
|
|
|
{
|
2012-04-02 01:38:43 +04:00
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
2010-09-16 01:06:33 +04:00
|
|
|
struct blkio_group *blkg;
|
|
|
|
struct hlist_node *n;
|
|
|
|
|
|
|
|
if (val < BLKIO_WEIGHT_MIN || val > BLKIO_WEIGHT_MAX)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
spin_lock(&blkio_list_lock);
|
|
|
|
spin_lock_irq(&blkcg->lock);
|
|
|
|
blkcg->weight = (unsigned int)val;
|
|
|
|
|
2012-03-06 01:15:16 +04:00
|
|
|
hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
|
2012-04-02 01:38:43 +04:00
|
|
|
struct blkg_policy_data *pd = blkg->pd[BLKIO_POLICY_PROP];
|
2012-03-06 01:15:16 +04:00
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
if (pd && !pd->conf.weight)
|
|
|
|
blkio_update_group_weight(blkg, BLKIO_POLICY_PROP,
|
|
|
|
blkcg->weight);
|
2012-03-06 01:15:16 +04:00
|
|
|
}
|
2010-09-16 01:06:33 +04:00
|
|
|
|
|
|
|
spin_unlock_irq(&blkcg->lock);
|
|
|
|
spin_unlock(&blkio_list_lock);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
/* for blk-throttle conf */
|
|
|
|
#ifdef CONFIG_BLK_DEV_THROTTLING
|
|
|
|
static u64 blkg_prfill_conf_u64(struct seq_file *sf,
|
|
|
|
struct blkg_policy_data *pd, int off)
|
|
|
|
{
|
|
|
|
u64 v = *(u64 *)((void *)&pd->conf + off);
|
|
|
|
|
|
|
|
if (!v)
|
|
|
|
return 0;
|
|
|
|
return __blkg_prfill_u64(sf, pd, v);
|
|
|
|
}
|
2010-09-16 01:06:33 +04:00
|
|
|
|
2012-04-02 01:38:43 +04:00
|
|
|
static int blkcg_print_conf_u64(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
struct seq_file *sf)
|
|
|
|
{
|
|
|
|
blkcg_print_blkgs(sf, cgroup_to_blkio_cgroup(cgrp),
|
|
|
|
blkg_prfill_conf_u64, BLKIO_POLICY_THROTL,
|
2012-04-02 01:38:43 +04:00
|
|
|
cft->private, false);
|
2010-09-16 01:06:33 +04:00
|
|
|
return 0;
|
|
|
|
}
|
2012-04-02 01:38:43 +04:00
|
|
|
|
|
|
|
static int blkcg_set_conf_u64(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
const char *buf, int rw,
|
|
|
|
void (*update)(struct blkio_group *, int, u64, int))
|
|
|
|
{
|
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgrp);
|
|
|
|
struct blkg_policy_data *pd;
|
|
|
|
struct blkg_conf_ctx ctx;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = blkg_conf_prep(blkcg, buf, &ctx);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
ret = -EINVAL;
|
|
|
|
pd = ctx.blkg->pd[BLKIO_POLICY_THROTL];
|
|
|
|
if (pd) {
|
|
|
|
*(u64 *)((void *)&pd->conf + cft->private) = ctx.v;
|
|
|
|
update(ctx.blkg, BLKIO_POLICY_THROTL, ctx.v ?: -1, rw);
|
|
|
|
ret = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
blkg_conf_finish(&ctx);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int blkcg_set_conf_bps_r(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
const char *buf)
|
|
|
|
{
|
|
|
|
return blkcg_set_conf_u64(cgrp, cft, buf, READ, blkio_update_group_bps);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int blkcg_set_conf_bps_w(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
const char *buf)
|
|
|
|
{
|
|
|
|
return blkcg_set_conf_u64(cgrp, cft, buf, WRITE, blkio_update_group_bps);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int blkcg_set_conf_iops_r(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
const char *buf)
|
|
|
|
{
|
|
|
|
return blkcg_set_conf_u64(cgrp, cft, buf, READ, blkio_update_group_iops);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int blkcg_set_conf_iops_w(struct cgroup *cgrp, struct cftype *cft,
|
|
|
|
const char *buf)
|
|
|
|
{
|
|
|
|
return blkcg_set_conf_u64(cgrp, cft, buf, WRITE, blkio_update_group_iops);
|
|
|
|
}
|
2012-04-02 01:38:43 +04:00
|
|
|
#endif
|
2010-09-16 01:06:33 +04:00
|
|
|
|
2009-12-03 20:59:42 +03:00
|
|
|
struct cftype blkio_files[] = {
|
2010-04-13 12:05:49 +04:00
|
|
|
{
|
|
|
|
.name = "weight_device",
|
2012-04-02 01:38:43 +04:00
|
|
|
.read_seq_string = blkcg_print_weight_device,
|
2012-04-02 01:38:43 +04:00
|
|
|
.write_string = blkcg_set_weight_device,
|
2010-04-13 12:05:49 +04:00
|
|
|
.max_write_len = 256,
|
|
|
|
},
|
2009-12-03 20:59:42 +03:00
|
|
|
{
|
|
|
|
.name = "weight",
|
2012-04-02 01:38:43 +04:00
|
|
|
.read_seq_string = blkcg_print_weight,
|
2012-04-02 01:38:43 +04:00
|
|
|
.write_u64 = blkcg_set_weight,
|
2009-12-03 20:59:42 +03:00
|
|
|
},
|
2009-12-03 20:59:49 +03:00
|
|
|
{
|
|
|
|
.name = "time",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats, time)),
|
|
|
|
.read_seq_string = blkcg_print_stat,
|
2009-12-03 20:59:49 +03:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.name = "sectors",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats_cpu, sectors)),
|
|
|
|
.read_seq_string = blkcg_print_cpu_stat,
|
2010-04-02 02:01:24 +04:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.name = "io_service_bytes",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats_cpu, service_bytes)),
|
|
|
|
.read_seq_string = blkcg_print_cpu_rwstat,
|
2010-04-02 02:01:24 +04:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.name = "io_serviced",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats_cpu, serviced)),
|
|
|
|
.read_seq_string = blkcg_print_cpu_rwstat,
|
2010-04-02 02:01:24 +04:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.name = "io_service_time",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats, service_time)),
|
|
|
|
.read_seq_string = blkcg_print_rwstat,
|
2010-04-02 02:01:24 +04:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.name = "io_wait_time",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats, wait_time)),
|
|
|
|
.read_seq_string = blkcg_print_rwstat,
|
2010-04-09 10:31:19 +04:00
|
|
|
},
|
2010-04-09 08:14:23 +04:00
|
|
|
{
|
|
|
|
.name = "io_merged",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats, merged)),
|
|
|
|
.read_seq_string = blkcg_print_rwstat,
|
2010-04-09 08:14:23 +04:00
|
|
|
},
|
2010-04-09 08:15:10 +04:00
|
|
|
{
|
|
|
|
.name = "io_queued",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats, queued)),
|
|
|
|
.read_seq_string = blkcg_print_rwstat,
|
2010-04-09 08:15:10 +04:00
|
|
|
},
|
2010-04-09 10:31:19 +04:00
|
|
|
{
|
|
|
|
.name = "reset_stats",
|
|
|
|
.write_u64 = blkiocg_reset_stats,
|
2009-12-03 20:59:49 +03:00
|
|
|
},
|
2010-10-01 16:49:41 +04:00
|
|
|
#ifdef CONFIG_BLK_DEV_THROTTLING
|
|
|
|
{
|
|
|
|
.name = "throttle.read_bps_device",
|
2012-04-02 01:38:43 +04:00
|
|
|
.private = offsetof(struct blkio_group_conf, bps[READ]),
|
2012-04-02 01:38:43 +04:00
|
|
|
.read_seq_string = blkcg_print_conf_u64,
|
2012-04-02 01:38:43 +04:00
|
|
|
.write_string = blkcg_set_conf_bps_r,
|
2010-10-01 16:49:41 +04:00
|
|
|
.max_write_len = 256,
|
|
|
|
},
|
|
|
|
|
|
|
|
{
|
|
|
|
.name = "throttle.write_bps_device",
|
2012-04-02 01:38:43 +04:00
|
|
|
.private = offsetof(struct blkio_group_conf, bps[WRITE]),
|
2012-04-02 01:38:43 +04:00
|
|
|
.read_seq_string = blkcg_print_conf_u64,
|
2012-04-02 01:38:43 +04:00
|
|
|
.write_string = blkcg_set_conf_bps_w,
|
2010-10-01 16:49:41 +04:00
|
|
|
.max_write_len = 256,
|
|
|
|
},
|
|
|
|
|
|
|
|
{
|
|
|
|
.name = "throttle.read_iops_device",
|
2012-04-02 01:38:43 +04:00
|
|
|
.private = offsetof(struct blkio_group_conf, iops[READ]),
|
2012-04-02 01:38:43 +04:00
|
|
|
.read_seq_string = blkcg_print_conf_u64,
|
2012-04-02 01:38:43 +04:00
|
|
|
.write_string = blkcg_set_conf_iops_r,
|
2010-10-01 16:49:41 +04:00
|
|
|
.max_write_len = 256,
|
|
|
|
},
|
|
|
|
|
|
|
|
{
|
|
|
|
.name = "throttle.write_iops_device",
|
2012-04-02 01:38:43 +04:00
|
|
|
.private = offsetof(struct blkio_group_conf, iops[WRITE]),
|
2012-04-02 01:38:43 +04:00
|
|
|
.read_seq_string = blkcg_print_conf_u64,
|
2012-04-02 01:38:43 +04:00
|
|
|
.write_string = blkcg_set_conf_iops_w,
|
2010-10-01 16:49:41 +04:00
|
|
|
.max_write_len = 256,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
.name = "throttle.io_service_bytes",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_THROTL,
|
|
|
|
offsetof(struct blkio_group_stats_cpu, service_bytes)),
|
|
|
|
.read_seq_string = blkcg_print_cpu_rwstat,
|
2010-10-01 16:49:41 +04:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.name = "throttle.io_serviced",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_THROTL,
|
|
|
|
offsetof(struct blkio_group_stats_cpu, serviced)),
|
|
|
|
.read_seq_string = blkcg_print_cpu_rwstat,
|
2010-10-01 16:49:41 +04:00
|
|
|
},
|
|
|
|
#endif /* CONFIG_BLK_DEV_THROTTLING */
|
|
|
|
|
2009-12-03 20:59:49 +03:00
|
|
|
#ifdef CONFIG_DEBUG_BLK_CGROUP
|
2010-04-09 08:15:10 +04:00
|
|
|
{
|
|
|
|
.name = "avg_queue_size",
|
2012-04-02 01:38:42 +04:00
|
|
|
.read_seq_string = blkcg_print_avg_queue_size,
|
2010-04-09 08:15:10 +04:00
|
|
|
},
|
2010-04-09 08:15:35 +04:00
|
|
|
{
|
|
|
|
.name = "group_wait_time",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats, group_wait_time)),
|
|
|
|
.read_seq_string = blkcg_print_stat,
|
2010-04-09 08:15:35 +04:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.name = "idle_time",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats, idle_time)),
|
|
|
|
.read_seq_string = blkcg_print_stat,
|
2010-04-09 08:15:35 +04:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.name = "empty_time",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats, empty_time)),
|
|
|
|
.read_seq_string = blkcg_print_stat,
|
2010-04-09 08:15:35 +04:00
|
|
|
},
|
2010-04-09 08:15:10 +04:00
|
|
|
{
|
2009-12-03 20:59:49 +03:00
|
|
|
.name = "dequeue",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats, dequeue)),
|
|
|
|
.read_seq_string = blkcg_print_stat,
|
2010-04-09 08:15:10 +04:00
|
|
|
},
|
2011-03-22 23:26:54 +03:00
|
|
|
{
|
|
|
|
.name = "unaccounted_time",
|
2012-04-02 01:38:42 +04:00
|
|
|
.private = BLKCG_STAT_PRIV(BLKIO_POLICY_PROP,
|
|
|
|
offsetof(struct blkio_group_stats, unaccounted_time)),
|
|
|
|
.read_seq_string = blkcg_print_stat,
|
2011-03-22 23:26:54 +03:00
|
|
|
},
|
2009-12-03 20:59:49 +03:00
|
|
|
#endif
|
2012-04-01 23:09:55 +04:00
|
|
|
{ } /* terminate */
|
2009-12-03 20:59:42 +03:00
|
|
|
};
|
|
|
|
|
2012-03-06 01:15:21 +04:00
|
|
|
/**
|
|
|
|
* blkiocg_pre_destroy - cgroup pre_destroy callback
|
|
|
|
* @cgroup: cgroup of interest
|
|
|
|
*
|
|
|
|
* This function is called when @cgroup is about to go away and responsible
|
|
|
|
* for shooting down all blkgs associated with @cgroup. blkgs should be
|
|
|
|
* removed while holding both q and blkcg locks. As blkcg lock is nested
|
|
|
|
* inside q lock, this function performs reverse double lock dancing.
|
|
|
|
*
|
|
|
|
* This is the blkcg counterpart of ioc_release_fn().
|
|
|
|
*/
|
2012-04-01 23:30:01 +04:00
|
|
|
static int blkiocg_pre_destroy(struct cgroup *cgroup)
|
2009-12-03 20:59:42 +03:00
|
|
|
{
|
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
|
2009-12-03 20:59:47 +03:00
|
|
|
|
2012-03-06 01:15:21 +04:00
|
|
|
spin_lock_irq(&blkcg->lock);
|
2012-03-06 01:15:11 +04:00
|
|
|
|
2012-03-06 01:15:21 +04:00
|
|
|
while (!hlist_empty(&blkcg->blkg_list)) {
|
|
|
|
struct blkio_group *blkg = hlist_entry(blkcg->blkg_list.first,
|
|
|
|
struct blkio_group, blkcg_node);
|
2012-03-06 01:15:22 +04:00
|
|
|
struct request_queue *q = blkg->q;
|
2009-12-03 20:59:47 +03:00
|
|
|
|
2012-03-06 01:15:21 +04:00
|
|
|
if (spin_trylock(q->queue_lock)) {
|
|
|
|
blkg_destroy(blkg);
|
|
|
|
spin_unlock(q->queue_lock);
|
|
|
|
} else {
|
|
|
|
spin_unlock_irq(&blkcg->lock);
|
|
|
|
cpu_relax();
|
2012-03-29 22:57:08 +04:00
|
|
|
spin_lock_irq(&blkcg->lock);
|
2010-05-03 16:28:55 +04:00
|
|
|
}
|
2012-03-06 01:15:21 +04:00
|
|
|
}
|
2009-12-03 20:59:47 +03:00
|
|
|
|
2012-03-06 01:15:21 +04:00
|
|
|
spin_unlock_irq(&blkcg->lock);
|
2012-03-06 01:15:11 +04:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2012-04-01 23:30:01 +04:00
|
|
|
static void blkiocg_destroy(struct cgroup *cgroup)
|
2012-03-06 01:15:11 +04:00
|
|
|
{
|
|
|
|
struct blkio_cgroup *blkcg = cgroup_to_blkio_cgroup(cgroup);
|
|
|
|
|
2010-03-11 02:22:11 +03:00
|
|
|
if (blkcg != &blkio_root_cgroup)
|
|
|
|
kfree(blkcg);
|
2009-12-03 20:59:42 +03:00
|
|
|
}
|
|
|
|
|
2012-01-31 09:47:36 +04:00
|
|
|
static struct cgroup_subsys_state *blkiocg_create(struct cgroup *cgroup)
|
2009-12-03 20:59:42 +03:00
|
|
|
{
|
2012-03-20 02:10:56 +04:00
|
|
|
static atomic64_t id_seq = ATOMIC64_INIT(0);
|
2010-05-07 10:57:00 +04:00
|
|
|
struct blkio_cgroup *blkcg;
|
|
|
|
struct cgroup *parent = cgroup->parent;
|
2009-12-03 20:59:42 +03:00
|
|
|
|
2010-05-07 10:57:00 +04:00
|
|
|
if (!parent) {
|
2009-12-03 20:59:42 +03:00
|
|
|
blkcg = &blkio_root_cgroup;
|
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
|
|
|
blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
|
|
|
|
if (!blkcg)
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
|
|
|
|
blkcg->weight = BLKIO_WEIGHT_DEFAULT;
|
2012-03-20 02:10:56 +04:00
|
|
|
blkcg->id = atomic64_inc_return(&id_seq); /* root is 0, start from 1 */
|
2009-12-03 20:59:42 +03:00
|
|
|
done:
|
|
|
|
spin_lock_init(&blkcg->lock);
|
|
|
|
INIT_HLIST_HEAD(&blkcg->blkg_list);
|
|
|
|
|
|
|
|
return &blkcg->css;
|
|
|
|
}
|
|
|
|
|
2012-03-06 01:15:12 +04:00
|
|
|
/**
|
|
|
|
* blkcg_init_queue - initialize blkcg part of request queue
|
|
|
|
* @q: request_queue to initialize
|
|
|
|
*
|
|
|
|
* Called from blk_alloc_queue_node(). Responsible for initializing blkcg
|
|
|
|
* part of new request_queue @q.
|
|
|
|
*
|
|
|
|
* RETURNS:
|
|
|
|
* 0 on success, -errno on failure.
|
|
|
|
*/
|
|
|
|
int blkcg_init_queue(struct request_queue *q)
|
|
|
|
{
|
2012-03-06 01:15:13 +04:00
|
|
|
int ret;
|
|
|
|
|
2012-03-06 01:15:12 +04:00
|
|
|
might_sleep();
|
|
|
|
|
2012-03-06 01:15:13 +04:00
|
|
|
ret = blk_throtl_init(q);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
mutex_lock(&all_q_mutex);
|
|
|
|
INIT_LIST_HEAD(&q->all_q_node);
|
|
|
|
list_add_tail(&q->all_q_node, &all_q_list);
|
|
|
|
mutex_unlock(&all_q_mutex);
|
|
|
|
|
|
|
|
return 0;
|
2012-03-06 01:15:12 +04:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* blkcg_drain_queue - drain blkcg part of request_queue
|
|
|
|
* @q: request_queue to drain
|
|
|
|
*
|
|
|
|
* Called from blk_drain_queue(). Responsible for draining blkcg part.
|
|
|
|
*/
|
|
|
|
void blkcg_drain_queue(struct request_queue *q)
|
|
|
|
{
|
|
|
|
lockdep_assert_held(q->queue_lock);
|
|
|
|
|
|
|
|
blk_throtl_drain(q);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* blkcg_exit_queue - exit and release blkcg part of request_queue
|
|
|
|
* @q: request_queue being released
|
|
|
|
*
|
|
|
|
* Called from blk_release_queue(). Responsible for exiting blkcg part.
|
|
|
|
*/
|
|
|
|
void blkcg_exit_queue(struct request_queue *q)
|
|
|
|
{
|
2012-03-06 01:15:13 +04:00
|
|
|
mutex_lock(&all_q_mutex);
|
|
|
|
list_del_init(&q->all_q_node);
|
|
|
|
mutex_unlock(&all_q_mutex);
|
|
|
|
|
2012-03-06 01:15:20 +04:00
|
|
|
blkg_destroy_all(q, true);
|
|
|
|
|
2012-03-06 01:15:12 +04:00
|
|
|
blk_throtl_exit(q);
|
|
|
|
}
|
|
|
|
|
2009-12-03 20:59:42 +03:00
|
|
|
/*
|
|
|
|
* We cannot support shared io contexts, as we have no mean to support
|
|
|
|
* two tasks with the same ioc in two different groups without major rework
|
|
|
|
* of the main cic data structures. For now we allow a task to change
|
|
|
|
* its cgroup only if it's the only owner of its ioc.
|
|
|
|
*/
|
2012-01-31 09:47:36 +04:00
|
|
|
static int blkiocg_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
|
2009-12-03 20:59:42 +03:00
|
|
|
{
|
2011-12-13 06:12:21 +04:00
|
|
|
struct task_struct *task;
|
2009-12-03 20:59:42 +03:00
|
|
|
struct io_context *ioc;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
/* task_lock() is needed to avoid races with exit_io_context() */
|
2011-12-13 06:12:21 +04:00
|
|
|
cgroup_taskset_for_each(task, cgrp, tset) {
|
|
|
|
task_lock(task);
|
|
|
|
ioc = task->io_context;
|
|
|
|
if (ioc && atomic_read(&ioc->nr_tasks) > 1)
|
|
|
|
ret = -EINVAL;
|
|
|
|
task_unlock(task);
|
|
|
|
if (ret)
|
|
|
|
break;
|
|
|
|
}
|
2009-12-03 20:59:42 +03:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2012-03-06 01:15:13 +04:00
|
|
|
static void blkcg_bypass_start(void)
|
|
|
|
__acquires(&all_q_mutex)
|
|
|
|
{
|
|
|
|
struct request_queue *q;
|
|
|
|
|
|
|
|
mutex_lock(&all_q_mutex);
|
|
|
|
|
|
|
|
list_for_each_entry(q, &all_q_list, all_q_node) {
|
|
|
|
blk_queue_bypass_start(q);
|
2012-03-06 01:15:20 +04:00
|
|
|
blkg_destroy_all(q, false);
|
2012-03-06 01:15:13 +04:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void blkcg_bypass_end(void)
|
|
|
|
__releases(&all_q_mutex)
|
|
|
|
{
|
|
|
|
struct request_queue *q;
|
|
|
|
|
|
|
|
list_for_each_entry(q, &all_q_list, all_q_node)
|
|
|
|
blk_queue_bypass_end(q);
|
|
|
|
|
|
|
|
mutex_unlock(&all_q_mutex);
|
|
|
|
}
|
|
|
|
|
2012-04-01 23:09:55 +04:00
|
|
|
struct cgroup_subsys blkio_subsys = {
|
|
|
|
.name = "blkio",
|
|
|
|
.create = blkiocg_create,
|
|
|
|
.can_attach = blkiocg_can_attach,
|
2012-04-01 23:30:01 +04:00
|
|
|
.pre_destroy = blkiocg_pre_destroy,
|
2012-04-01 23:09:55 +04:00
|
|
|
.destroy = blkiocg_destroy,
|
|
|
|
.subsys_id = blkio_subsys_id,
|
2012-04-01 23:09:55 +04:00
|
|
|
.base_cftypes = blkio_files,
|
2012-04-01 23:09:55 +04:00
|
|
|
.module = THIS_MODULE,
|
|
|
|
};
|
|
|
|
EXPORT_SYMBOL_GPL(blkio_subsys);
|
|
|
|
|
2009-12-04 18:36:42 +03:00
|
|
|
void blkio_policy_register(struct blkio_policy_type *blkiop)
|
|
|
|
{
|
2012-03-06 01:15:20 +04:00
|
|
|
struct request_queue *q;
|
|
|
|
|
2012-03-06 01:15:13 +04:00
|
|
|
blkcg_bypass_start();
|
2009-12-04 18:36:42 +03:00
|
|
|
spin_lock(&blkio_list_lock);
|
2012-03-06 01:15:04 +04:00
|
|
|
|
|
|
|
BUG_ON(blkio_policy[blkiop->plid]);
|
|
|
|
blkio_policy[blkiop->plid] = blkiop;
|
2009-12-04 18:36:42 +03:00
|
|
|
list_add_tail(&blkiop->list, &blkio_list);
|
2012-03-06 01:15:04 +04:00
|
|
|
|
2009-12-04 18:36:42 +03:00
|
|
|
spin_unlock(&blkio_list_lock);
|
2012-03-06 01:15:20 +04:00
|
|
|
list_for_each_entry(q, &all_q_list, all_q_node)
|
|
|
|
update_root_blkg_pd(q, blkiop->plid);
|
2012-03-06 01:15:13 +04:00
|
|
|
blkcg_bypass_end();
|
2009-12-04 18:36:42 +03:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blkio_policy_register);
|
|
|
|
|
|
|
|
void blkio_policy_unregister(struct blkio_policy_type *blkiop)
|
|
|
|
{
|
2012-03-06 01:15:20 +04:00
|
|
|
struct request_queue *q;
|
|
|
|
|
2012-03-06 01:15:13 +04:00
|
|
|
blkcg_bypass_start();
|
2009-12-04 18:36:42 +03:00
|
|
|
spin_lock(&blkio_list_lock);
|
2012-03-06 01:15:04 +04:00
|
|
|
|
|
|
|
BUG_ON(blkio_policy[blkiop->plid] != blkiop);
|
|
|
|
blkio_policy[blkiop->plid] = NULL;
|
2009-12-04 18:36:42 +03:00
|
|
|
list_del_init(&blkiop->list);
|
2012-03-06 01:15:04 +04:00
|
|
|
|
2009-12-04 18:36:42 +03:00
|
|
|
spin_unlock(&blkio_list_lock);
|
2012-03-06 01:15:20 +04:00
|
|
|
list_for_each_entry(q, &all_q_list, all_q_node)
|
|
|
|
update_root_blkg_pd(q, blkiop->plid);
|
2012-03-06 01:15:13 +04:00
|
|
|
blkcg_bypass_end();
|
2009-12-04 18:36:42 +03:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(blkio_policy_unregister);
|