diff options
author | Jens Axboe <axboe@fb.com> | 2015-04-17 22:15:18 (GMT) |
---|---|---|
committer | Jens Axboe <axboe@fb.com> | 2015-05-05 19:32:47 (GMT) |
commit | c4cf5261f8bffd9de132b50660a69148e7575bd6 (patch) | |
tree | c3ff2d0d2db5f7392e8651b4f0d2958895faf8b2 /drivers/md | |
parent | d9cee5d4f66ef36f69b0108dedbad7f7009bb6a8 (diff) | |
download | linux-c4cf5261f8bffd9de132b50660a69148e7575bd6.tar.xz |
bio: skip atomic inc/dec of ->bi_remaining for non-chains
Struct bio has an atomic ref count for chained bio's, and we use this
to know when to end IO on the bio. However, most bio's are not chained,
so we don't need to always introduce this atomic operation as part of
ending IO.
Add a helper to elevate the bi_remaining count, and flag the bio as
now actually needing the decrement at end_io time. Rename the field
to __bi_remaining to catch any current users of this doing the
incrementing manually.
For high IOPS workloads, this reduces the overhead of bio_endio()
substantially.
Tested-by: Robert Elliott <elliott@hp.com>
Acked-by: Kent Overstreet <kent.overstreet@gmail.com>
Reviewed-by: Jan Kara <jack@suse.cz>
Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'drivers/md')
-rw-r--r-- | drivers/md/dm-cache-target.c | 2 | ||||
-rw-r--r-- | drivers/md/dm-raid1.c | 2 | ||||
-rw-r--r-- | drivers/md/dm-snap.c | 2 | ||||
-rw-r--r-- | drivers/md/dm-thin.c | 4 |
4 files changed, 5 insertions, 5 deletions
diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index 7755af3..705eb7b 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -91,7 +91,7 @@ static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio) * Must bump bi_remaining to allow bio to complete with * restored bi_end_io. */ - atomic_inc(&bio->bi_remaining); + bio_inc_remaining(bio); } /*----------------------------------------------------------------*/ diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index 089d627..d6a1c09 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -1254,7 +1254,7 @@ static int mirror_end_io(struct dm_target *ti, struct bio *bio, int error) dm_bio_restore(bd, bio); bio_record->details.bi_bdev = NULL; - atomic_inc(&bio->bi_remaining); + bio_inc_remaining(bio); queue_bio(ms, bio, rw); return DM_ENDIO_INCOMPLETE; diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index f83a0f3..8bfeae2 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c @@ -1478,7 +1478,7 @@ out: if (full_bio) { full_bio->bi_end_io = pe->full_bio_end_io; full_bio->bi_private = pe->full_bio_private; - atomic_inc(&full_bio->bi_remaining); + bio_inc_remaining(full_bio); } increment_pending_exceptions_done_count(); diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index 921aafd..342dbda 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -795,7 +795,7 @@ static void process_prepared_mapping_fail(struct dm_thin_new_mapping *m) { if (m->bio) { m->bio->bi_end_io = m->saved_bi_end_io; - atomic_inc(&m->bio->bi_remaining); + bio_inc_remaining(m->bio); } cell_error(m->tc->pool, m->cell); list_del(&m->list); @@ -812,7 +812,7 @@ static void process_prepared_mapping(struct dm_thin_new_mapping *m) bio = m->bio; if (bio) { bio->bi_end_io = m->saved_bi_end_io; - atomic_inc(&bio->bi_remaining); + bio_inc_remaining(bio); } if (m->err) { |