bio-integrity: stop abusing bi_end_io

And instead call directly into the integrity code from bio_end_io.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/block/bio-integrity.c b/block/bio-integrity.c
index 8df4eb103ba9e78c089a1d1dd2c9cdb35a89d30a..f733beab6ca2b2dd6cce017ddaff06caa103fbda 100644 (file)
--- a/block/bio-integrity.c
+++ b/block/bio-integrity.c
@@ -102,7 +102,7 @@ EXPORT_SYMBOL(bio_integrity_alloc);
  * Description: Used to free the integrity portion of a bio. Usually
  * called from bio_free().
  */
-void bio_integrity_free(struct bio *bio)
+static void bio_integrity_free(struct bio *bio)
 {
        struct bio_integrity_payload *bip = bio_integrity(bio);
        struct bio_set *bs = bio->bi_pool;
@@ -120,8 +120,8 @@ void bio_integrity_free(struct bio *bio)
        }
 
        bio->bi_integrity = NULL;
+       bio->bi_opf &= ~REQ_INTEGRITY;
 }
-EXPORT_SYMBOL(bio_integrity_free);
 
 /**
  * bio_integrity_add_page - Attach integrity metadata
@@ -326,12 +326,6 @@ bool bio_integrity_prep(struct bio *bio)
                offset = 0;
        }
 
-       /* Install custom I/O completion handler if read verify is enabled */
-       if (bio_data_dir(bio) == READ) {
-               bip->bip_end_io = bio->bi_end_io;
-               bio->bi_end_io = bio_integrity_endio;
-       }
-
        /* Auto-generate integrity metadata if this is a write */
        if (bio_data_dir(bio) == WRITE) {
                bio_integrity_process(bio, &bio->bi_iter,
@@ -375,13 +369,12 @@ static void bio_integrity_verify_fn(struct work_struct *work)
                bio->bi_status = BLK_STS_IOERR;
        }
 
-       /* Restore original bio completion handler */
-       bio->bi_end_io = bip->bip_end_io;
+       bio_integrity_free(bio);
        bio_endio(bio);
 }
 
 /**
- * bio_integrity_endio - Integrity I/O completion function
+ * __bio_integrity_endio - Integrity I/O completion function
  * @bio:       Protected bio
  * @error:     Pointer to errno
  *
@@ -392,27 +385,19 @@ static void bio_integrity_verify_fn(struct work_struct *work)
  * in process context. This function postpones completion
  * accordingly.
  */
-void bio_integrity_endio(struct bio *bio)
+bool __bio_integrity_endio(struct bio *bio)
 {
-       struct bio_integrity_payload *bip = bio_integrity(bio);
+       if (bio_op(bio) == REQ_OP_READ && !bio->bi_status) {
+               struct bio_integrity_payload *bip = bio_integrity(bio);
 
-       BUG_ON(bip->bip_bio != bio);
-
-       /* In case of an I/O error there is no point in verifying the
-        * integrity metadata.  Restore original bio end_io handler
-        * and run it.
-        */
-       if (bio->bi_status) {
-               bio->bi_end_io = bip->bip_end_io;
-               bio_endio(bio);
-
-               return;
+               INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);
+               queue_work(kintegrityd_wq, &bip->bip_work);
+               return false;
        }
 
-       INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);
-       queue_work(kintegrityd_wq, &bip->bip_work);
+       bio_integrity_free(bio);
+       return true;
 }
-EXPORT_SYMBOL(bio_integrity_endio);
 
 /**
  * bio_integrity_advance - Advance integrity vector

diff --git a/block/bio.c b/block/bio.c
index a6b225324a61b21854497ea09ef36b555f9a09d5..9cabf5d0be2096bfb9b3ab0a60323d91defff3d3 100644 (file)
--- a/block/bio.c
+++ b/block/bio.c
@@ -243,9 +243,6 @@ fallback:
 void bio_uninit(struct bio *bio)
 {
        bio_disassociate_task(bio);
-
-       if (bio_integrity(bio))
-               bio_integrity_free(bio);
 }
 EXPORT_SYMBOL(bio_uninit);
 
@@ -1813,6 +1810,8 @@ void bio_endio(struct bio *bio)
 again:
        if (!bio_remaining_done(bio))
                return;
+       if (!bio_integrity_endio(bio))
+               return;
 
        /*
         * Need to have a real endio function for chained bios, otherwise

diff --git a/block/blk.h b/block/blk.h
index 01ebb8185f6bfac8eb827ebdaee3a9d6845660dc..3a3d715bd7253b82522b30fcde33161f909dcf0b 100644 (file)
--- a/block/blk.h
+++ b/block/blk.h
@@ -81,10 +81,21 @@ static inline void blk_queue_enter_live(struct request_queue *q)
 
 #ifdef CONFIG_BLK_DEV_INTEGRITY
 void blk_flush_integrity(void);
+bool __bio_integrity_endio(struct bio *);
+static inline bool bio_integrity_endio(struct bio *bio)
+{
+       if (bio_integrity(bio))
+               return __bio_integrity_endio(bio);
+       return true;
+}
 #else
 static inline void blk_flush_integrity(void)
 {
 }
+static inline bool bio_integrity_endio(struct bio *bio)
+{
+       return true;
+}
 #endif
 
 void blk_timeout_work(struct work_struct *work);

diff --git a/include/linux/bio.h b/include/linux/bio.h
index 1eba19580185e4b9893a3cbc8156baa9dec89a2c..7b1cf4ba090291af59e904dc1c59d8196ab88d3d 100644 (file)
--- a/include/linux/bio.h
+++ b/include/linux/bio.h
@@ -320,8 +320,6 @@ struct bio_integrity_payload {
 
        struct bvec_iter        bip_iter;
 
-       bio_end_io_t            *bip_end_io;    /* saved I/O completion fn */
-
        unsigned short          bip_slab;       /* slab the bip came from */
        unsigned short          bip_vcnt;       /* # of integrity bio_vecs */
        unsigned short          bip_max_vcnt;   /* integrity bio_vec slots */
@@ -739,10 +737,8 @@ struct biovec_slab {
                bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
 
 extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
-extern void bio_integrity_free(struct bio *);
 extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
 extern bool bio_integrity_prep(struct bio *);
-extern void bio_integrity_endio(struct bio *);
 extern void bio_integrity_advance(struct bio *, unsigned int);
 extern void bio_integrity_trim(struct bio *);
 extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
@@ -772,11 +768,6 @@ static inline bool bio_integrity_prep(struct bio *bio)
        return true;
 }
 
-static inline void bio_integrity_free(struct bio *bio)
-{
-       return;
-}
-
 static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
                                      gfp_t gfp_mask)
 {