2 * Copyright (C) 2012 Red Hat, Inc.
4 * Author: Mikulas Patocka <mpatocka@redhat.com>
6 * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
8 * This file is released under the GPLv2.
10 * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
11 * default prefetch value. Data are read in "prefetch_cluster" chunks from the
12 * hash device. Setting this greatly improves performance when data and hash
13 * are on the same disk on different partitions on devices with poor random
17 #include "dm-verity.h"
18 #include "dm-verity-fec.h"
20 #include <linux/module.h>
21 #include <linux/reboot.h>
23 #define DM_MSG_PREFIX "verity"
25 #define DM_VERITY_ENV_LENGTH 42
26 #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR"
28 #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
30 #define DM_VERITY_MAX_CORRUPTED_ERRS 100
32 #define DM_VERITY_OPT_LOGGING "ignore_corruption"
33 #define DM_VERITY_OPT_RESTART "restart_on_corruption"
34 #define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks"
35 #define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once"
37 #define DM_VERITY_OPTS_MAX (2 + DM_VERITY_OPTS_FEC)
39 static unsigned dm_verity_prefetch_cluster
= DM_VERITY_DEFAULT_PREFETCH_SIZE
;
41 module_param_named(prefetch_cluster
, dm_verity_prefetch_cluster
, uint
, S_IRUGO
| S_IWUSR
);
43 struct dm_verity_prefetch_work
{
44 struct work_struct work
;
51 * Auxiliary structure appended to each dm-bufio buffer. If the value
52 * hash_verified is nonzero, hash of the block has been verified.
54 * The variable hash_verified is set to 0 when allocating the buffer, then
55 * it can be changed to 1 and it is never reset to 0 again.
57 * There is no lock around this value, a race condition can at worst cause
58 * that multiple processes verify the hash of the same buffer simultaneously
59 * and write 1 to hash_verified simultaneously.
60 * This condition is harmless, so we don't need locking.
67 * Initialize struct buffer_aux for a freshly created buffer.
69 static void dm_bufio_alloc_callback(struct dm_buffer
*buf
)
71 struct buffer_aux
*aux
= dm_bufio_get_aux_data(buf
);
73 aux
->hash_verified
= 0;
77 * Translate input sector number to the sector number on the target device.
79 static sector_t
verity_map_sector(struct dm_verity
*v
, sector_t bi_sector
)
81 return v
->data_start
+ dm_target_offset(v
->ti
, bi_sector
);
85 * Return hash position of a specified block at a specified tree level
86 * (0 is the lowest level).
87 * The lowest "hash_per_block_bits"-bits of the result denote hash position
88 * inside a hash block. The remaining bits denote location of the hash block.
90 static sector_t
verity_position_at_level(struct dm_verity
*v
, sector_t block
,
93 return block
>> (level
* v
->hash_per_block_bits
);
97 * Callback function for asynchrnous crypto API completion notification
99 static void verity_op_done(struct crypto_async_request
*base
, int err
)
101 struct verity_result
*res
= (struct verity_result
*)base
->data
;
103 if (err
== -EINPROGRESS
)
107 complete(&res
->completion
);
111 * Wait for async crypto API callback
113 static inline int verity_complete_op(struct verity_result
*res
, int ret
)
121 ret
= wait_for_completion_interruptible(&res
->completion
);
124 reinit_completion(&res
->completion
);
128 DMERR("verity_wait_hash: crypto op submission failed: %d", ret
);
131 if (unlikely(ret
< 0))
132 DMERR("verity_wait_hash: crypto op failed: %d", ret
);
137 static int verity_hash_update(struct dm_verity
*v
, struct ahash_request
*req
,
138 const u8
*data
, size_t len
,
139 struct verity_result
*res
)
141 struct scatterlist sg
;
143 sg_init_one(&sg
, data
, len
);
144 ahash_request_set_crypt(req
, &sg
, NULL
, len
);
146 return verity_complete_op(res
, crypto_ahash_update(req
));
150 * Wrapper for crypto_ahash_init, which handles verity salting.
152 static int verity_hash_init(struct dm_verity
*v
, struct ahash_request
*req
,
153 struct verity_result
*res
)
157 ahash_request_set_tfm(req
, v
->tfm
);
158 ahash_request_set_callback(req
, CRYPTO_TFM_REQ_MAY_SLEEP
|
159 CRYPTO_TFM_REQ_MAY_BACKLOG
,
160 verity_op_done
, (void *)res
);
161 init_completion(&res
->completion
);
163 r
= verity_complete_op(res
, crypto_ahash_init(req
));
165 if (unlikely(r
< 0)) {
166 DMERR("crypto_ahash_init failed: %d", r
);
170 if (likely(v
->salt_size
&& (v
->version
>= 1)))
171 r
= verity_hash_update(v
, req
, v
->salt
, v
->salt_size
, res
);
176 static int verity_hash_final(struct dm_verity
*v
, struct ahash_request
*req
,
177 u8
*digest
, struct verity_result
*res
)
181 if (unlikely(v
->salt_size
&& (!v
->version
))) {
182 r
= verity_hash_update(v
, req
, v
->salt
, v
->salt_size
, res
);
185 DMERR("verity_hash_final failed updating salt: %d", r
);
190 ahash_request_set_crypt(req
, NULL
, digest
, 0);
191 r
= verity_complete_op(res
, crypto_ahash_final(req
));
196 int verity_hash(struct dm_verity
*v
, struct ahash_request
*req
,
197 const u8
*data
, size_t len
, u8
*digest
)
200 struct verity_result res
;
202 r
= verity_hash_init(v
, req
, &res
);
206 r
= verity_hash_update(v
, req
, data
, len
, &res
);
210 r
= verity_hash_final(v
, req
, digest
, &res
);
216 static void verity_hash_at_level(struct dm_verity
*v
, sector_t block
, int level
,
217 sector_t
*hash_block
, unsigned *offset
)
219 sector_t position
= verity_position_at_level(v
, block
, level
);
222 *hash_block
= v
->hash_level_block
[level
] + (position
>> v
->hash_per_block_bits
);
227 idx
= position
& ((1 << v
->hash_per_block_bits
) - 1);
229 *offset
= idx
* v
->digest_size
;
231 *offset
= idx
<< (v
->hash_dev_block_bits
- v
->hash_per_block_bits
);
235 * Handle verification errors.
237 static int verity_handle_err(struct dm_verity
*v
, enum verity_block_type type
,
238 unsigned long long block
)
240 char verity_env
[DM_VERITY_ENV_LENGTH
];
241 char *envp
[] = { verity_env
, NULL
};
242 const char *type_str
= "";
243 struct mapped_device
*md
= dm_table_get_md(v
->ti
->table
);
245 /* Corruption should be visible in device status in all modes */
248 if (v
->corrupted_errs
>= DM_VERITY_MAX_CORRUPTED_ERRS
)
254 case DM_VERITY_BLOCK_TYPE_DATA
:
257 case DM_VERITY_BLOCK_TYPE_METADATA
:
258 type_str
= "metadata";
264 DMERR("%s: %s block %llu is corrupted", v
->data_dev
->name
, type_str
,
267 if (v
->corrupted_errs
== DM_VERITY_MAX_CORRUPTED_ERRS
)
268 DMERR("%s: reached maximum errors", v
->data_dev
->name
);
270 snprintf(verity_env
, DM_VERITY_ENV_LENGTH
, "%s=%d,%llu",
271 DM_VERITY_ENV_VAR_NAME
, type
, block
);
273 kobject_uevent_env(&disk_to_dev(dm_disk(md
))->kobj
, KOBJ_CHANGE
, envp
);
276 if (v
->mode
== DM_VERITY_MODE_LOGGING
)
279 if (v
->mode
== DM_VERITY_MODE_RESTART
)
280 kernel_restart("dm-verity device corrupted");
286 * Verify hash of a metadata block pertaining to the specified data block
287 * ("block" argument) at a specified level ("level" argument).
289 * On successful return, verity_io_want_digest(v, io) contains the hash value
290 * for a lower tree level or for the data block (if we're at the lowest level).
292 * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.
293 * If "skip_unverified" is false, unverified buffer is hashed and verified
294 * against current value of verity_io_want_digest(v, io).
296 static int verity_verify_level(struct dm_verity
*v
, struct dm_verity_io
*io
,
297 sector_t block
, int level
, bool skip_unverified
,
300 struct dm_buffer
*buf
;
301 struct buffer_aux
*aux
;
307 verity_hash_at_level(v
, block
, level
, &hash_block
, &offset
);
309 data
= dm_bufio_read(v
->bufio
, hash_block
, &buf
);
311 return PTR_ERR(data
);
313 aux
= dm_bufio_get_aux_data(buf
);
315 if (!aux
->hash_verified
) {
316 if (skip_unverified
) {
321 r
= verity_hash(v
, verity_io_hash_req(v
, io
),
322 data
, 1 << v
->hash_dev_block_bits
,
323 verity_io_real_digest(v
, io
));
327 if (likely(memcmp(verity_io_real_digest(v
, io
), want_digest
,
328 v
->digest_size
) == 0))
329 aux
->hash_verified
= 1;
330 else if (verity_fec_decode(v
, io
,
331 DM_VERITY_BLOCK_TYPE_METADATA
,
332 hash_block
, data
, NULL
) == 0)
333 aux
->hash_verified
= 1;
334 else if (verity_handle_err(v
,
335 DM_VERITY_BLOCK_TYPE_METADATA
,
343 memcpy(want_digest
, data
, v
->digest_size
);
347 dm_bufio_release(buf
);
352 * Find a hash for a given block, write it to digest and verify the integrity
353 * of the hash tree if necessary.
355 int verity_hash_for_block(struct dm_verity
*v
, struct dm_verity_io
*io
,
356 sector_t block
, u8
*digest
, bool *is_zero
)
360 if (likely(v
->levels
)) {
362 * First, we try to get the requested hash for
363 * the current block. If the hash block itself is
364 * verified, zero is returned. If it isn't, this
365 * function returns 1 and we fall back to whole
366 * chain verification.
368 r
= verity_verify_level(v
, io
, block
, 0, true, digest
);
373 memcpy(digest
, v
->root_digest
, v
->digest_size
);
375 for (i
= v
->levels
- 1; i
>= 0; i
--) {
376 r
= verity_verify_level(v
, io
, block
, i
, false, digest
);
381 if (!r
&& v
->zero_digest
)
382 *is_zero
= !memcmp(v
->zero_digest
, digest
, v
->digest_size
);
390 * Calculates the digest for the given bio
392 int verity_for_io_block(struct dm_verity
*v
, struct dm_verity_io
*io
,
393 struct bvec_iter
*iter
, struct verity_result
*res
)
395 unsigned int todo
= 1 << v
->data_dev_block_bits
;
396 struct bio
*bio
= dm_bio_from_per_bio_data(io
, v
->ti
->per_io_data_size
);
397 struct scatterlist sg
;
398 struct ahash_request
*req
= verity_io_hash_req(v
, io
);
403 struct bio_vec bv
= bio_iter_iovec(bio
, *iter
);
405 sg_init_table(&sg
, 1);
409 if (likely(len
>= todo
))
412 * Operating on a single page at a time looks suboptimal
413 * until you consider the typical block size is 4,096B.
414 * Going through this loops twice should be very rare.
416 sg_set_page(&sg
, bv
.bv_page
, len
, bv
.bv_offset
);
417 ahash_request_set_crypt(req
, &sg
, NULL
, len
);
418 r
= verity_complete_op(res
, crypto_ahash_update(req
));
420 if (unlikely(r
< 0)) {
421 DMERR("verity_for_io_block crypto op failed: %d", r
);
425 bio_advance_iter(bio
, iter
, len
);
433 * Calls function process for 1 << v->data_dev_block_bits bytes in the bio_vec
434 * starting from iter.
436 int verity_for_bv_block(struct dm_verity
*v
, struct dm_verity_io
*io
,
437 struct bvec_iter
*iter
,
438 int (*process
)(struct dm_verity
*v
,
439 struct dm_verity_io
*io
, u8
*data
,
442 unsigned todo
= 1 << v
->data_dev_block_bits
;
443 struct bio
*bio
= dm_bio_from_per_bio_data(io
, v
->ti
->per_io_data_size
);
449 struct bio_vec bv
= bio_iter_iovec(bio
, *iter
);
451 page
= kmap_atomic(bv
.bv_page
);
454 if (likely(len
>= todo
))
457 r
= process(v
, io
, page
+ bv
.bv_offset
, len
);
463 bio_advance_iter(bio
, iter
, len
);
470 static int verity_bv_zero(struct dm_verity
*v
, struct dm_verity_io
*io
,
471 u8
*data
, size_t len
)
473 memset(data
, 0, len
);
478 * Moves the bio iter one data block forward.
480 static inline void verity_bv_skip_block(struct dm_verity
*v
,
481 struct dm_verity_io
*io
,
482 struct bvec_iter
*iter
)
484 struct bio
*bio
= dm_bio_from_per_bio_data(io
, v
->ti
->per_io_data_size
);
486 bio_advance_iter(bio
, iter
, 1 << v
->data_dev_block_bits
);
490 * Verify one "dm_verity_io" structure.
492 static int verity_verify_io(struct dm_verity_io
*io
)
495 struct dm_verity
*v
= io
->v
;
496 struct bvec_iter start
;
498 struct verity_result res
;
500 for (b
= 0; b
< io
->n_blocks
; b
++) {
502 sector_t cur_block
= io
->block
+ b
;
503 struct ahash_request
*req
= verity_io_hash_req(v
, io
);
505 if (v
->validated_blocks
&&
506 likely(test_bit(cur_block
, v
->validated_blocks
))) {
507 verity_bv_skip_block(v
, io
, &io
->iter
);
511 r
= verity_hash_for_block(v
, io
, cur_block
,
512 verity_io_want_digest(v
, io
),
519 * If we expect a zero block, don't validate, just
522 r
= verity_for_bv_block(v
, io
, &io
->iter
,
530 r
= verity_hash_init(v
, req
, &res
);
535 r
= verity_for_io_block(v
, io
, &io
->iter
, &res
);
539 r
= verity_hash_final(v
, req
, verity_io_real_digest(v
, io
),
544 if (likely(memcmp(verity_io_real_digest(v
, io
),
545 verity_io_want_digest(v
, io
), v
->digest_size
) == 0)) {
546 if (v
->validated_blocks
)
547 set_bit(cur_block
, v
->validated_blocks
);
550 else if (verity_fec_decode(v
, io
, DM_VERITY_BLOCK_TYPE_DATA
,
551 cur_block
, NULL
, &start
) == 0)
553 else if (verity_handle_err(v
, DM_VERITY_BLOCK_TYPE_DATA
,
562 * End one "io" structure with a given error.
564 static void verity_finish_io(struct dm_verity_io
*io
, blk_status_t status
)
566 struct dm_verity
*v
= io
->v
;
567 struct bio
*bio
= dm_bio_from_per_bio_data(io
, v
->ti
->per_io_data_size
);
569 bio
->bi_end_io
= io
->orig_bi_end_io
;
570 bio
->bi_status
= status
;
572 verity_fec_finish_io(io
);
577 static void verity_work(struct work_struct
*w
)
579 struct dm_verity_io
*io
= container_of(w
, struct dm_verity_io
, work
);
581 verity_finish_io(io
, errno_to_blk_status(verity_verify_io(io
)));
584 static void verity_end_io(struct bio
*bio
)
586 struct dm_verity_io
*io
= bio
->bi_private
;
588 if (bio
->bi_status
&& !verity_fec_is_enabled(io
->v
)) {
589 verity_finish_io(io
, bio
->bi_status
);
593 INIT_WORK(&io
->work
, verity_work
);
594 queue_work(io
->v
->verify_wq
, &io
->work
);
598 * Prefetch buffers for the specified io.
599 * The root buffer is not prefetched, it is assumed that it will be cached
602 static void verity_prefetch_io(struct work_struct
*work
)
604 struct dm_verity_prefetch_work
*pw
=
605 container_of(work
, struct dm_verity_prefetch_work
, work
);
606 struct dm_verity
*v
= pw
->v
;
608 sector_t prefetch_size
;
610 for (i
= v
->levels
- 2; i
>= 0; i
--) {
611 sector_t hash_block_start
;
612 sector_t hash_block_end
;
613 verity_hash_at_level(v
, pw
->block
, i
, &hash_block_start
, NULL
);
614 verity_hash_at_level(v
, pw
->block
+ pw
->n_blocks
- 1, i
, &hash_block_end
, NULL
);
616 unsigned cluster
= ACCESS_ONCE(dm_verity_prefetch_cluster
);
618 cluster
>>= v
->data_dev_block_bits
;
619 if (unlikely(!cluster
))
620 goto no_prefetch_cluster
;
622 if (unlikely(cluster
& (cluster
- 1)))
623 cluster
= 1 << __fls(cluster
);
625 hash_block_start
&= ~(sector_t
)(cluster
- 1);
626 hash_block_end
|= cluster
- 1;
627 if (unlikely(hash_block_end
>= v
->hash_blocks
))
628 hash_block_end
= v
->hash_blocks
- 1;
631 // for emmc, it is more efficient to send bigger read
632 prefetch_size
= max((sector_t
)CONFIG_DM_VERITY_HASH_PREFETCH_MIN_SIZE
,
633 hash_block_end
- hash_block_start
+ 1);
634 if ((hash_block_start
+ prefetch_size
) >= (v
->hash_start
+ v
->hash_blocks
)) {
635 prefetch_size
= hash_block_end
- hash_block_start
+ 1;
637 dm_bufio_prefetch(v
->bufio
, hash_block_start
,
644 static void verity_submit_prefetch(struct dm_verity
*v
, struct dm_verity_io
*io
)
646 struct dm_verity_prefetch_work
*pw
;
648 pw
= kmalloc(sizeof(struct dm_verity_prefetch_work
),
649 GFP_NOIO
| __GFP_NORETRY
| __GFP_NOMEMALLOC
| __GFP_NOWARN
);
654 INIT_WORK(&pw
->work
, verity_prefetch_io
);
656 pw
->block
= io
->block
;
657 pw
->n_blocks
= io
->n_blocks
;
658 queue_work(v
->verify_wq
, &pw
->work
);
662 * Bio map function. It allocates dm_verity_io structure and bio vector and
663 * fills them. Then it issues prefetches and the I/O.
665 int verity_map(struct dm_target
*ti
, struct bio
*bio
)
667 struct dm_verity
*v
= ti
->private;
668 struct dm_verity_io
*io
;
670 bio_set_dev(bio
, v
->data_dev
->bdev
);
671 bio
->bi_iter
.bi_sector
= verity_map_sector(v
, bio
->bi_iter
.bi_sector
);
673 if (((unsigned)bio
->bi_iter
.bi_sector
| bio_sectors(bio
)) &
674 ((1 << (v
->data_dev_block_bits
- SECTOR_SHIFT
)) - 1)) {
675 DMERR_LIMIT("unaligned io");
676 return DM_MAPIO_KILL
;
679 if (bio_end_sector(bio
) >>
680 (v
->data_dev_block_bits
- SECTOR_SHIFT
) > v
->data_blocks
) {
681 DMERR_LIMIT("io out of range");
682 return DM_MAPIO_KILL
;
685 if (bio_data_dir(bio
) == WRITE
)
686 return DM_MAPIO_KILL
;
688 io
= dm_per_bio_data(bio
, ti
->per_io_data_size
);
690 io
->orig_bi_end_io
= bio
->bi_end_io
;
691 io
->block
= bio
->bi_iter
.bi_sector
>> (v
->data_dev_block_bits
- SECTOR_SHIFT
);
692 io
->n_blocks
= bio
->bi_iter
.bi_size
>> v
->data_dev_block_bits
;
694 bio
->bi_end_io
= verity_end_io
;
695 bio
->bi_private
= io
;
696 io
->iter
= bio
->bi_iter
;
698 verity_fec_init_io(io
);
700 verity_submit_prefetch(v
, io
);
702 generic_make_request(bio
);
704 return DM_MAPIO_SUBMITTED
;
708 * Status: V (valid) or C (corruption found)
710 void verity_status(struct dm_target
*ti
, status_type_t type
,
711 unsigned status_flags
, char *result
, unsigned maxlen
)
713 struct dm_verity
*v
= ti
->private;
719 case STATUSTYPE_INFO
:
720 DMEMIT("%c", v
->hash_failed
? 'C' : 'V');
722 case STATUSTYPE_TABLE
:
723 DMEMIT("%u %s %s %u %u %llu %llu %s ",
727 1 << v
->data_dev_block_bits
,
728 1 << v
->hash_dev_block_bits
,
729 (unsigned long long)v
->data_blocks
,
730 (unsigned long long)v
->hash_start
,
733 for (x
= 0; x
< v
->digest_size
; x
++)
734 DMEMIT("%02x", v
->root_digest
[x
]);
739 for (x
= 0; x
< v
->salt_size
; x
++)
740 DMEMIT("%02x", v
->salt
[x
]);
741 if (v
->mode
!= DM_VERITY_MODE_EIO
)
743 if (verity_fec_is_enabled(v
))
744 args
+= DM_VERITY_OPTS_FEC
;
747 if (v
->validated_blocks
)
752 if (v
->mode
!= DM_VERITY_MODE_EIO
) {
755 case DM_VERITY_MODE_LOGGING
:
756 DMEMIT(DM_VERITY_OPT_LOGGING
);
758 case DM_VERITY_MODE_RESTART
:
759 DMEMIT(DM_VERITY_OPT_RESTART
);
766 DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES
);
767 if (v
->validated_blocks
)
768 DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE
);
769 sz
= verity_fec_status_table(v
, sz
, result
, maxlen
);
774 int verity_prepare_ioctl(struct dm_target
*ti
,
775 struct block_device
**bdev
, fmode_t
*mode
)
777 struct dm_verity
*v
= ti
->private;
779 *bdev
= v
->data_dev
->bdev
;
782 ti
->len
!= i_size_read(v
->data_dev
->bdev
->bd_inode
) >> SECTOR_SHIFT
)
787 int verity_iterate_devices(struct dm_target
*ti
,
788 iterate_devices_callout_fn fn
, void *data
)
790 struct dm_verity
*v
= ti
->private;
792 return fn(ti
, v
->data_dev
, v
->data_start
, ti
->len
, data
);
795 void verity_io_hints(struct dm_target
*ti
, struct queue_limits
*limits
)
797 struct dm_verity
*v
= ti
->private;
799 if (limits
->logical_block_size
< 1 << v
->data_dev_block_bits
)
800 limits
->logical_block_size
= 1 << v
->data_dev_block_bits
;
802 if (limits
->physical_block_size
< 1 << v
->data_dev_block_bits
)
803 limits
->physical_block_size
= 1 << v
->data_dev_block_bits
;
805 blk_limits_io_min(limits
, limits
->logical_block_size
);
808 void verity_dtr(struct dm_target
*ti
)
810 struct dm_verity
*v
= ti
->private;
813 destroy_workqueue(v
->verify_wq
);
816 dm_bufio_client_destroy(v
->bufio
);
818 kvfree(v
->validated_blocks
);
820 kfree(v
->root_digest
);
821 kfree(v
->zero_digest
);
824 crypto_free_ahash(v
->tfm
);
829 dm_put_device(ti
, v
->hash_dev
);
832 dm_put_device(ti
, v
->data_dev
);
839 static int verity_alloc_most_once(struct dm_verity
*v
)
841 struct dm_target
*ti
= v
->ti
;
843 /* the bitset can only handle INT_MAX blocks */
844 if (v
->data_blocks
> INT_MAX
) {
845 ti
->error
= "device too large to use check_at_most_once";
849 v
->validated_blocks
= kvzalloc(BITS_TO_LONGS(v
->data_blocks
) *
850 sizeof(unsigned long), GFP_KERNEL
);
851 if (!v
->validated_blocks
) {
852 ti
->error
= "failed to allocate bitset for check_at_most_once";
859 static int verity_alloc_zero_digest(struct dm_verity
*v
)
862 struct ahash_request
*req
;
865 v
->zero_digest
= kmalloc(v
->digest_size
, GFP_KERNEL
);
870 req
= kmalloc(v
->ahash_reqsize
, GFP_KERNEL
);
873 return r
; /* verity_dtr will free zero_digest */
875 zero_data
= kzalloc(1 << v
->data_dev_block_bits
, GFP_KERNEL
);
880 r
= verity_hash(v
, req
, zero_data
, 1 << v
->data_dev_block_bits
,
890 static int verity_parse_opt_args(struct dm_arg_set
*as
, struct dm_verity
*v
)
894 struct dm_target
*ti
= v
->ti
;
895 const char *arg_name
;
897 static const struct dm_arg _args
[] = {
898 {0, DM_VERITY_OPTS_MAX
, "Invalid number of feature args"},
901 r
= dm_read_arg_group(_args
, as
, &argc
, &ti
->error
);
909 arg_name
= dm_shift_arg(as
);
912 if (!strcasecmp(arg_name
, DM_VERITY_OPT_LOGGING
)) {
913 v
->mode
= DM_VERITY_MODE_LOGGING
;
916 } else if (!strcasecmp(arg_name
, DM_VERITY_OPT_RESTART
)) {
917 v
->mode
= DM_VERITY_MODE_RESTART
;
920 } else if (!strcasecmp(arg_name
, DM_VERITY_OPT_IGN_ZEROES
)) {
921 r
= verity_alloc_zero_digest(v
);
923 ti
->error
= "Cannot allocate zero digest";
928 } else if (!strcasecmp(arg_name
, DM_VERITY_OPT_AT_MOST_ONCE
)) {
929 r
= verity_alloc_most_once(v
);
934 } else if (verity_is_fec_opt_arg(arg_name
)) {
935 r
= verity_fec_parse_opt_args(as
, v
, &argc
, arg_name
);
941 ti
->error
= "Unrecognized verity feature request";
943 } while (argc
&& !r
);
950 * <version> The current format is version 1.
951 * Vsn 0 is compatible with original Chromium OS releases.
956 * <the number of data blocks>
960 * <salt> Hex string or "-" if no salt.
962 int verity_ctr(struct dm_target
*ti
, unsigned argc
, char **argv
)
965 struct dm_arg_set as
;
967 unsigned long long num_ll
;
970 sector_t hash_position
;
973 v
= kzalloc(sizeof(struct dm_verity
), GFP_KERNEL
);
975 ti
->error
= "Cannot allocate verity structure";
981 r
= verity_fec_ctr_alloc(v
);
985 if ((dm_table_get_mode(ti
->table
) & ~FMODE_READ
)) {
986 ti
->error
= "Device must be readonly";
992 ti
->error
= "Not enough arguments";
997 if (sscanf(argv
[0], "%u%c", &num
, &dummy
) != 1 ||
999 ti
->error
= "Invalid version";
1005 r
= dm_get_device(ti
, argv
[1], FMODE_READ
, &v
->data_dev
);
1007 ti
->error
= "Data device lookup failed";
1011 r
= dm_get_device(ti
, argv
[2], FMODE_READ
, &v
->hash_dev
);
1013 ti
->error
= "Hash device lookup failed";
1017 if (sscanf(argv
[3], "%u%c", &num
, &dummy
) != 1 ||
1018 !num
|| (num
& (num
- 1)) ||
1019 num
< bdev_logical_block_size(v
->data_dev
->bdev
) ||
1021 ti
->error
= "Invalid data device block size";
1025 v
->data_dev_block_bits
= __ffs(num
);
1027 if (sscanf(argv
[4], "%u%c", &num
, &dummy
) != 1 ||
1028 !num
|| (num
& (num
- 1)) ||
1029 num
< bdev_logical_block_size(v
->hash_dev
->bdev
) ||
1031 ti
->error
= "Invalid hash device block size";
1035 v
->hash_dev_block_bits
= __ffs(num
);
1037 if (sscanf(argv
[5], "%llu%c", &num_ll
, &dummy
) != 1 ||
1038 (sector_t
)(num_ll
<< (v
->data_dev_block_bits
- SECTOR_SHIFT
))
1039 >> (v
->data_dev_block_bits
- SECTOR_SHIFT
) != num_ll
) {
1040 ti
->error
= "Invalid data blocks";
1044 v
->data_blocks
= num_ll
;
1046 if (ti
->len
> (v
->data_blocks
<< (v
->data_dev_block_bits
- SECTOR_SHIFT
))) {
1047 ti
->error
= "Data device is too small";
1052 if (sscanf(argv
[6], "%llu%c", &num_ll
, &dummy
) != 1 ||
1053 (sector_t
)(num_ll
<< (v
->hash_dev_block_bits
- SECTOR_SHIFT
))
1054 >> (v
->hash_dev_block_bits
- SECTOR_SHIFT
) != num_ll
) {
1055 ti
->error
= "Invalid hash start";
1059 v
->hash_start
= num_ll
;
1061 v
->alg_name
= kstrdup(argv
[7], GFP_KERNEL
);
1063 ti
->error
= "Cannot allocate algorithm name";
1068 v
->tfm
= crypto_alloc_ahash(v
->alg_name
, 0, 0);
1069 if (IS_ERR(v
->tfm
)) {
1070 ti
->error
= "Cannot initialize hash function";
1071 r
= PTR_ERR(v
->tfm
);
1075 v
->digest_size
= crypto_ahash_digestsize(v
->tfm
);
1076 if ((1 << v
->hash_dev_block_bits
) < v
->digest_size
* 2) {
1077 ti
->error
= "Digest size too big";
1081 v
->ahash_reqsize
= sizeof(struct ahash_request
) +
1082 crypto_ahash_reqsize(v
->tfm
);
1084 v
->root_digest
= kmalloc(v
->digest_size
, GFP_KERNEL
);
1085 if (!v
->root_digest
) {
1086 ti
->error
= "Cannot allocate root digest";
1090 if (strlen(argv
[8]) != v
->digest_size
* 2 ||
1091 hex2bin(v
->root_digest
, argv
[8], v
->digest_size
)) {
1092 ti
->error
= "Invalid root digest";
1097 if (strcmp(argv
[9], "-")) {
1098 v
->salt_size
= strlen(argv
[9]) / 2;
1099 v
->salt
= kmalloc(v
->salt_size
, GFP_KERNEL
);
1101 ti
->error
= "Cannot allocate salt";
1105 if (strlen(argv
[9]) != v
->salt_size
* 2 ||
1106 hex2bin(v
->salt
, argv
[9], v
->salt_size
)) {
1107 ti
->error
= "Invalid salt";
1116 /* Optional parameters */
1121 r
= verity_parse_opt_args(&as
, v
);
1126 v
->hash_per_block_bits
=
1127 __fls((1 << v
->hash_dev_block_bits
) / v
->digest_size
);
1131 while (v
->hash_per_block_bits
* v
->levels
< 64 &&
1132 (unsigned long long)(v
->data_blocks
- 1) >>
1133 (v
->hash_per_block_bits
* v
->levels
))
1136 if (v
->levels
> DM_VERITY_MAX_LEVELS
) {
1137 ti
->error
= "Too many tree levels";
1142 hash_position
= v
->hash_start
;
1143 for (i
= v
->levels
- 1; i
>= 0; i
--) {
1145 v
->hash_level_block
[i
] = hash_position
;
1146 s
= (v
->data_blocks
+ ((sector_t
)1 << ((i
+ 1) * v
->hash_per_block_bits
)) - 1)
1147 >> ((i
+ 1) * v
->hash_per_block_bits
);
1148 if (hash_position
+ s
< hash_position
) {
1149 ti
->error
= "Hash device offset overflow";
1155 v
->hash_blocks
= hash_position
;
1157 v
->bufio
= dm_bufio_client_create(v
->hash_dev
->bdev
,
1158 1 << v
->hash_dev_block_bits
, 1, sizeof(struct buffer_aux
),
1159 dm_bufio_alloc_callback
, NULL
);
1160 if (IS_ERR(v
->bufio
)) {
1161 ti
->error
= "Cannot initialize dm-bufio";
1162 r
= PTR_ERR(v
->bufio
);
1167 if (dm_bufio_get_device_size(v
->bufio
) < v
->hash_blocks
) {
1168 ti
->error
= "Hash device is too small";
1173 /* WQ_UNBOUND greatly improves performance when running on ramdisk */
1174 v
->verify_wq
= alloc_workqueue("kverityd", WQ_CPU_INTENSIVE
| WQ_MEM_RECLAIM
| WQ_UNBOUND
, num_online_cpus());
1175 if (!v
->verify_wq
) {
1176 ti
->error
= "Cannot allocate workqueue";
1181 ti
->per_io_data_size
= sizeof(struct dm_verity_io
) +
1182 v
->ahash_reqsize
+ v
->digest_size
* 2;
1184 r
= verity_fec_ctr(v
);
1188 ti
->per_io_data_size
= roundup(ti
->per_io_data_size
,
1189 __alignof__(struct dm_verity_io
));
1199 static struct target_type verity_target
= {
1201 .version
= {1, 4, 0},
1202 .module
= THIS_MODULE
,
1206 .status
= verity_status
,
1207 .prepare_ioctl
= verity_prepare_ioctl
,
1208 .iterate_devices
= verity_iterate_devices
,
1209 .io_hints
= verity_io_hints
,
1212 static int __init
dm_verity_init(void)
1216 r
= dm_register_target(&verity_target
);
1218 DMERR("register failed %d", r
);
1223 static void __exit
dm_verity_exit(void)
1225 dm_unregister_target(&verity_target
);
1228 module_init(dm_verity_init
);
1229 module_exit(dm_verity_exit
);
1231 MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
1232 MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
1233 MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
1234 MODULE_DESCRIPTION(DM_NAME
" target for transparent disk integrity checking");
1235 MODULE_LICENSE("GPL");