projects / GitHub / LineageOS / android_kernel_motorola_exynos9610.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
defconfig: exynos9610: Re-add dropped Wi-Fi AP options lost
[GitHub/LineageOS/android_kernel_motorola_exynos9610.git] / drivers / md / dm-cache-target.c
... / ...
CommitLineData
1/*
2 * Copyright (C) 2012 Red Hat. All rights reserved.
3 *
4 * This file is released under the GPL.
5 */
6
7#include "dm.h"
8#include "dm-bio-prison-v2.h"
9#include "dm-bio-record.h"
10#include "dm-cache-metadata.h"
11
12#include <linux/dm-io.h>
13#include <linux/dm-kcopyd.h>
14#include <linux/jiffies.h>
15#include <linux/init.h>
16#include <linux/mempool.h>
17#include <linux/module.h>
18#include <linux/rwsem.h>
19#include <linux/slab.h>
20#include <linux/vmalloc.h>
21
22#define DM_MSG_PREFIX "cache"
23
24DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(cache_copy_throttle,
25 "A percentage of time allocated for copying to and/or from cache");
26
27/*----------------------------------------------------------------*/
28
29/*
30 * Glossary:
31 *
32 * oblock: index of an origin block
33 * cblock: index of a cache block
34 * promotion: movement of a block from origin to cache
35 * demotion: movement of a block from cache to origin
36 * migration: movement of a block between the origin and cache device,
37 * either direction
38 */
39
40/*----------------------------------------------------------------*/
41
42struct io_tracker {
43 spinlock_t lock;
44
45 /*
46 * Sectors of in-flight IO.
47 */
48 sector_t in_flight;
49
50 /*
51 * The time, in jiffies, when this device became idle (if it is
52 * indeed idle).
53 */
54 unsigned long idle_time;
55 unsigned long last_update_time;
56};
57
58static void iot_init(struct io_tracker *iot)
59{
60 spin_lock_init(&iot->lock);
61 iot->in_flight = 0ul;
62 iot->idle_time = 0ul;
63 iot->last_update_time = jiffies;
64}
65
66static bool __iot_idle_for(struct io_tracker *iot, unsigned long jifs)
67{
68 if (iot->in_flight)
69 return false;
70
71 return time_after(jiffies, iot->idle_time + jifs);
72}
73
74static bool iot_idle_for(struct io_tracker *iot, unsigned long jifs)
75{
76 bool r;
77 unsigned long flags;
78
79 spin_lock_irqsave(&iot->lock, flags);
80 r = __iot_idle_for(iot, jifs);
81 spin_unlock_irqrestore(&iot->lock, flags);
82
83 return r;
84}
85
86static void iot_io_begin(struct io_tracker *iot, sector_t len)
87{
88 unsigned long flags;
89
90 spin_lock_irqsave(&iot->lock, flags);
91 iot->in_flight += len;
92 spin_unlock_irqrestore(&iot->lock, flags);
93}
94
95static void __iot_io_end(struct io_tracker *iot, sector_t len)
96{
97 if (!len)
98 return;
99
100 iot->in_flight -= len;
101 if (!iot->in_flight)
102 iot->idle_time = jiffies;
103}
104
105static void iot_io_end(struct io_tracker *iot, sector_t len)
106{
107 unsigned long flags;
108
109 spin_lock_irqsave(&iot->lock, flags);
110 __iot_io_end(iot, len);
111 spin_unlock_irqrestore(&iot->lock, flags);
112}
113
114/*----------------------------------------------------------------*/
115
116/*
117 * Represents a chunk of future work. 'input' allows continuations to pass
118 * values between themselves, typically error values.
119 */
120struct continuation {
121 struct work_struct ws;
122 blk_status_t input;
123};
124
125static inline void init_continuation(struct continuation *k,
126 void (*fn)(struct work_struct *))
127{
128 INIT_WORK(&k->ws, fn);
129 k->input = 0;
130}
131
132static inline void queue_continuation(struct workqueue_struct *wq,
133 struct continuation *k)
134{
135 queue_work(wq, &k->ws);
136}
137
138/*----------------------------------------------------------------*/
139
140/*
141 * The batcher collects together pieces of work that need a particular
142 * operation to occur before they can proceed (typically a commit).
143 */
144struct batcher {
145 /*
146 * The operation that everyone is waiting for.
147 */
148 blk_status_t (*commit_op)(void *context);
149 void *commit_context;
150
151 /*
152 * This is how bios should be issued once the commit op is complete
153 * (accounted_request).
154 */
155 void (*issue_op)(struct bio *bio, void *context);
156 void *issue_context;
157
158 /*
159 * Queued work gets put on here after commit.
160 */
161 struct workqueue_struct *wq;
162
163 spinlock_t lock;
164 struct list_head work_items;
165 struct bio_list bios;
166 struct work_struct commit_work;
167
168 bool commit_scheduled;
169};
170
171static void __commit(struct work_struct *_ws)
172{
173 struct batcher *b = container_of(_ws, struct batcher, commit_work);
174 blk_status_t r;
175 unsigned long flags;
176 struct list_head work_items;
177 struct work_struct *ws, *tmp;
178 struct continuation *k;
179 struct bio *bio;
180 struct bio_list bios;
181
182 INIT_LIST_HEAD(&work_items);
183 bio_list_init(&bios);
184
185 /*
186 * We have to grab these before the commit_op to avoid a race
187 * condition.
188 */
189 spin_lock_irqsave(&b->lock, flags);
190 list_splice_init(&b->work_items, &work_items);
191 bio_list_merge(&bios, &b->bios);
192 bio_list_init(&b->bios);
193 b->commit_scheduled = false;
194 spin_unlock_irqrestore(&b->lock, flags);
195
196 r = b->commit_op(b->commit_context);
197
198 list_for_each_entry_safe(ws, tmp, &work_items, entry) {
199 k = container_of(ws, struct continuation, ws);
200 k->input = r;
201 INIT_LIST_HEAD(&ws->entry); /* to avoid a WARN_ON */
202 queue_work(b->wq, ws);
203 }
204
205 while ((bio = bio_list_pop(&bios))) {
206 if (r) {
207 bio->bi_status = r;
208 bio_endio(bio);
209 } else
210 b->issue_op(bio, b->issue_context);
211 }
212}
213
214static void batcher_init(struct batcher *b,
215 blk_status_t (*commit_op)(void *),
216 void *commit_context,
217 void (*issue_op)(struct bio *bio, void *),
218 void *issue_context,
219 struct workqueue_struct *wq)
220{
221 b->commit_op = commit_op;
222 b->commit_context = commit_context;
223 b->issue_op = issue_op;
224 b->issue_context = issue_context;
225 b->wq = wq;
226
227 spin_lock_init(&b->lock);
228 INIT_LIST_HEAD(&b->work_items);
229 bio_list_init(&b->bios);
230 INIT_WORK(&b->commit_work, __commit);
231 b->commit_scheduled = false;
232}
233
234static void async_commit(struct batcher *b)
235{
236 queue_work(b->wq, &b->commit_work);
237}
238
239static void continue_after_commit(struct batcher *b, struct continuation *k)
240{
241 unsigned long flags;
242 bool commit_scheduled;
243
244 spin_lock_irqsave(&b->lock, flags);
245 commit_scheduled = b->commit_scheduled;
246 list_add_tail(&k->ws.entry, &b->work_items);
247 spin_unlock_irqrestore(&b->lock, flags);
248
249 if (commit_scheduled)
250 async_commit(b);
251}
252
253/*
254 * Bios are errored if commit failed.
255 */
256static void issue_after_commit(struct batcher *b, struct bio *bio)
257{
258 unsigned long flags;
259 bool commit_scheduled;
260
261 spin_lock_irqsave(&b->lock, flags);
262 commit_scheduled = b->commit_scheduled;
263 bio_list_add(&b->bios, bio);
264 spin_unlock_irqrestore(&b->lock, flags);
265
266 if (commit_scheduled)
267 async_commit(b);
268}
269
270/*
271 * Call this if some urgent work is waiting for the commit to complete.
272 */
273static void schedule_commit(struct batcher *b)
274{
275 bool immediate;
276 unsigned long flags;
277
278 spin_lock_irqsave(&b->lock, flags);
279 immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios);
280 b->commit_scheduled = true;
281 spin_unlock_irqrestore(&b->lock, flags);
282
283 if (immediate)
284 async_commit(b);
285}
286
287/*
288 * There are a couple of places where we let a bio run, but want to do some
289 * work before calling its endio function. We do this by temporarily
290 * changing the endio fn.
291 */
292struct dm_hook_info {
293 bio_end_io_t *bi_end_io;
294};
295
296static void dm_hook_bio(struct dm_hook_info *h, struct bio *bio,
297 bio_end_io_t *bi_end_io, void *bi_private)
298{
299 h->bi_end_io = bio->bi_end_io;
300
301 bio->bi_end_io = bi_end_io;
302 bio->bi_private = bi_private;
303}
304
305static void dm_unhook_bio(struct dm_hook_info *h, struct bio *bio)
306{
307 bio->bi_end_io = h->bi_end_io;
308}
309
310/*----------------------------------------------------------------*/
311
312#define MIGRATION_POOL_SIZE 128
313#define COMMIT_PERIOD HZ
314#define MIGRATION_COUNT_WINDOW 10
315
316/*
317 * The block size of the device holding cache data must be
318 * between 32KB and 1GB.
319 */
320#define DATA_DEV_BLOCK_SIZE_MIN_SECTORS (32 * 1024 >> SECTOR_SHIFT)
321#define DATA_DEV_BLOCK_SIZE_MAX_SECTORS (1024 * 1024 * 1024 >> SECTOR_SHIFT)
322
323enum cache_metadata_mode {
324 CM_WRITE, /* metadata may be changed */
325 CM_READ_ONLY, /* metadata may not be changed */
326 CM_FAIL
327};
328
329enum cache_io_mode {
330 /*
331 * Data is written to cached blocks only. These blocks are marked
332 * dirty. If you lose the cache device you will lose data.
333 * Potential performance increase for both reads and writes.
334 */
335 CM_IO_WRITEBACK,
336
337 /*
338 * Data is written to both cache and origin. Blocks are never
339 * dirty. Potential performance benfit for reads only.
340 */
341 CM_IO_WRITETHROUGH,
342
343 /*
344 * A degraded mode useful for various cache coherency situations
345 * (eg, rolling back snapshots). Reads and writes always go to the
346 * origin. If a write goes to a cached oblock, then the cache
347 * block is invalidated.
348 */
349 CM_IO_PASSTHROUGH
350};
351
352struct cache_features {
353 enum cache_metadata_mode mode;
354 enum cache_io_mode io_mode;
355 unsigned metadata_version;
356};
357
358struct cache_stats {
359 atomic_t read_hit;
360 atomic_t read_miss;
361 atomic_t write_hit;
362 atomic_t write_miss;
363 atomic_t demotion;
364 atomic_t promotion;
365 atomic_t writeback;
366 atomic_t copies_avoided;
367 atomic_t cache_cell_clash;
368 atomic_t commit_count;
369 atomic_t discard_count;
370};
371
372struct cache {
373 struct dm_target *ti;
374 struct dm_target_callbacks callbacks;
375
376 struct dm_cache_metadata *cmd;
377
378 /*
379 * Metadata is written to this device.
380 */
381 struct dm_dev *metadata_dev;
382
383 /*
384 * The slower of the two data devices. Typically a spindle.
385 */
386 struct dm_dev *origin_dev;
387
388 /*
389 * The faster of the two data devices. Typically an SSD.
390 */
391 struct dm_dev *cache_dev;
392
393 /*
394 * Size of the origin device in _complete_ blocks and native sectors.
395 */
396 dm_oblock_t origin_blocks;
397 sector_t origin_sectors;
398
399 /*
400 * Size of the cache device in blocks.
401 */
402 dm_cblock_t cache_size;
403
404 /*
405 * Fields for converting from sectors to blocks.
406 */
407 sector_t sectors_per_block;
408 int sectors_per_block_shift;
409
410 spinlock_t lock;
411 struct list_head deferred_cells;
412 struct bio_list deferred_bios;
413 struct bio_list deferred_writethrough_bios;
414 sector_t migration_threshold;
415 wait_queue_head_t migration_wait;
416 atomic_t nr_allocated_migrations;
417
418 /*
419 * The number of in flight migrations that are performing
420 * background io. eg, promotion, writeback.
421 */
422 atomic_t nr_io_migrations;
423
424 struct rw_semaphore quiesce_lock;
425
426 /*
427 * cache_size entries, dirty if set
428 */
429 atomic_t nr_dirty;
430 unsigned long *dirty_bitset;
431
432 /*
433 * origin_blocks entries, discarded if set.
434 */
435 dm_dblock_t discard_nr_blocks;
436 unsigned long *discard_bitset;
437 uint32_t discard_block_size; /* a power of 2 times sectors per block */
438
439 /*
440 * Rather than reconstructing the table line for the status we just
441 * save it and regurgitate.
442 */
443 unsigned nr_ctr_args;
444 const char **ctr_args;
445
446 struct dm_kcopyd_client *copier;
447 struct workqueue_struct *wq;
448 struct work_struct deferred_bio_worker;
449 struct work_struct deferred_writethrough_worker;
450 struct work_struct migration_worker;
451 struct delayed_work waker;
452 struct dm_bio_prison_v2 *prison;
453
454 mempool_t *migration_pool;
455
456 struct dm_cache_policy *policy;
457 unsigned policy_nr_args;
458
459 bool need_tick_bio:1;
460 bool sized:1;
461 bool invalidate:1;
462 bool commit_requested:1;
463 bool loaded_mappings:1;
464 bool loaded_discards:1;
465
466 /*
467 * Cache features such as write-through.
468 */
469 struct cache_features features;
470
471 struct cache_stats stats;
472
473 /*
474 * Invalidation fields.
475 */
476 spinlock_t invalidation_lock;
477 struct list_head invalidation_requests;
478
479 struct io_tracker tracker;
480
481 struct work_struct commit_ws;
482 struct batcher committer;
483
484 struct rw_semaphore background_work_lock;
485};
486
487struct per_bio_data {
488 bool tick:1;
489 unsigned req_nr:2;
490 struct dm_bio_prison_cell_v2 *cell;
491 struct dm_hook_info hook_info;
492 sector_t len;
493
494 /*
495 * writethrough fields. These MUST remain at the end of this
496 * structure and the 'cache' member must be the first as it
497 * is used to determine the offset of the writethrough fields.
498 */
499 struct cache *cache;
500 dm_cblock_t cblock;
501 struct dm_bio_details bio_details;
502};
503
504struct dm_cache_migration {
505 struct continuation k;
506 struct cache *cache;
507
508 struct policy_work *op;
509 struct bio *overwrite_bio;
510 struct dm_bio_prison_cell_v2 *cell;
511
512 dm_cblock_t invalidate_cblock;
513 dm_oblock_t invalidate_oblock;
514};
515
516/*----------------------------------------------------------------*/
517
518static bool writethrough_mode(struct cache_features *f)
519{
520 return f->io_mode == CM_IO_WRITETHROUGH;
521}
522
523static bool writeback_mode(struct cache_features *f)
524{
525 return f->io_mode == CM_IO_WRITEBACK;
526}
527
528static inline bool passthrough_mode(struct cache_features *f)
529{
530 return unlikely(f->io_mode == CM_IO_PASSTHROUGH);
531}
532
533/*----------------------------------------------------------------*/
534
535static void wake_deferred_bio_worker(struct cache *cache)
536{
537 queue_work(cache->wq, &cache->deferred_bio_worker);
538}
539
540static void wake_deferred_writethrough_worker(struct cache *cache)
541{
542 queue_work(cache->wq, &cache->deferred_writethrough_worker);
543}
544
545static void wake_migration_worker(struct cache *cache)
546{
547 if (passthrough_mode(&cache->features))
548 return;
549
550 queue_work(cache->wq, &cache->migration_worker);
551}
552
553/*----------------------------------------------------------------*/
554
555static struct dm_bio_prison_cell_v2 *alloc_prison_cell(struct cache *cache)
556{
557 return dm_bio_prison_alloc_cell_v2(cache->prison, GFP_NOWAIT);
558}
559
560static void free_prison_cell(struct cache *cache, struct dm_bio_prison_cell_v2 *cell)
561{
562 dm_bio_prison_free_cell_v2(cache->prison, cell);
563}
564
565static struct dm_cache_migration *alloc_migration(struct cache *cache)
566{
567 struct dm_cache_migration *mg;
568
569 mg = mempool_alloc(cache->migration_pool, GFP_NOWAIT);
570 if (mg) {
571 mg->cache = cache;
572 atomic_inc(&mg->cache->nr_allocated_migrations);
573 }
574
575 return mg;
576}
577
578static void free_migration(struct dm_cache_migration *mg)
579{
580 struct cache *cache = mg->cache;
581
582 if (atomic_dec_and_test(&cache->nr_allocated_migrations))
583 wake_up(&cache->migration_wait);
584
585 mempool_free(mg, cache->migration_pool);
586}
587
588/*----------------------------------------------------------------*/
589
590static inline dm_oblock_t oblock_succ(dm_oblock_t b)
591{
592 return to_oblock(from_oblock(b) + 1ull);
593}
594
595static void build_key(dm_oblock_t begin, dm_oblock_t end, struct dm_cell_key_v2 *key)
596{
597 key->virtual = 0;
598 key->dev = 0;
599 key->block_begin = from_oblock(begin);
600 key->block_end = from_oblock(end);
601}
602
603/*
604 * We have two lock levels. Level 0, which is used to prevent WRITEs, and
605 * level 1 which prevents *both* READs and WRITEs.
606 */
607#define WRITE_LOCK_LEVEL 0
608#define READ_WRITE_LOCK_LEVEL 1
609
610static unsigned lock_level(struct bio *bio)
611{
612 return bio_data_dir(bio) == WRITE ?
613 WRITE_LOCK_LEVEL :
614 READ_WRITE_LOCK_LEVEL;
615}
616
617/*----------------------------------------------------------------
618 * Per bio data
619 *--------------------------------------------------------------*/
620
621/*
622 * If using writeback, leave out struct per_bio_data's writethrough fields.
623 */
624#define PB_DATA_SIZE_WB (offsetof(struct per_bio_data, cache))
625#define PB_DATA_SIZE_WT (sizeof(struct per_bio_data))
626
627static size_t get_per_bio_data_size(struct cache *cache)
628{
629 return writethrough_mode(&cache->features) ? PB_DATA_SIZE_WT : PB_DATA_SIZE_WB;
630}
631
632static struct per_bio_data *get_per_bio_data(struct bio *bio, size_t data_size)
633{
634 struct per_bio_data *pb = dm_per_bio_data(bio, data_size);
635 BUG_ON(!pb);
636 return pb;
637}
638
639static struct per_bio_data *init_per_bio_data(struct bio *bio, size_t data_size)
640{
641 struct per_bio_data *pb = get_per_bio_data(bio, data_size);
642
643 pb->tick = false;
644 pb->req_nr = dm_bio_get_target_bio_nr(bio);
645 pb->cell = NULL;
646 pb->len = 0;
647
648 return pb;
649}
650
651/*----------------------------------------------------------------*/
652
653static void defer_bio(struct cache *cache, struct bio *bio)
654{
655 unsigned long flags;
656
657 spin_lock_irqsave(&cache->lock, flags);
658 bio_list_add(&cache->deferred_bios, bio);
659 spin_unlock_irqrestore(&cache->lock, flags);
660
661 wake_deferred_bio_worker(cache);
662}
663
664static void defer_bios(struct cache *cache, struct bio_list *bios)
665{
666 unsigned long flags;
667
668 spin_lock_irqsave(&cache->lock, flags);
669 bio_list_merge(&cache->deferred_bios, bios);
670 bio_list_init(bios);
671 spin_unlock_irqrestore(&cache->lock, flags);
672
673 wake_deferred_bio_worker(cache);
674}
675
676/*----------------------------------------------------------------*/
677
678static bool bio_detain_shared(struct cache *cache, dm_oblock_t oblock, struct bio *bio)
679{
680 bool r;
681 size_t pb_size;
682 struct per_bio_data *pb;
683 struct dm_cell_key_v2 key;
684 dm_oblock_t end = to_oblock(from_oblock(oblock) + 1ULL);
685 struct dm_bio_prison_cell_v2 *cell_prealloc, *cell;
686
687 cell_prealloc = alloc_prison_cell(cache); /* FIXME: allow wait if calling from worker */
688 if (!cell_prealloc) {
689 defer_bio(cache, bio);
690 return false;
691 }
692
693 build_key(oblock, end, &key);
694 r = dm_cell_get_v2(cache->prison, &key, lock_level(bio), bio, cell_prealloc, &cell);
695 if (!r) {
696 /*
697 * Failed to get the lock.
698 */
699 free_prison_cell(cache, cell_prealloc);
700 return r;
701 }
702
703 if (cell != cell_prealloc)
704 free_prison_cell(cache, cell_prealloc);
705
706 pb_size = get_per_bio_data_size(cache);
707 pb = get_per_bio_data(bio, pb_size);
708 pb->cell = cell;
709
710 return r;
711}
712
713/*----------------------------------------------------------------*/
714
715static bool is_dirty(struct cache *cache, dm_cblock_t b)
716{
717 return test_bit(from_cblock(b), cache->dirty_bitset);
718}
719
720static void set_dirty(struct cache *cache, dm_cblock_t cblock)
721{
722 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset)) {
723 atomic_inc(&cache->nr_dirty);
724 policy_set_dirty(cache->policy, cblock);
725 }
726}
727
728/*
729 * These two are called when setting after migrations to force the policy
730 * and dirty bitset to be in sync.
731 */
732static void force_set_dirty(struct cache *cache, dm_cblock_t cblock)
733{
734 if (!test_and_set_bit(from_cblock(cblock), cache->dirty_bitset))
735 atomic_inc(&cache->nr_dirty);
736 policy_set_dirty(cache->policy, cblock);
737}
738
739static void force_clear_dirty(struct cache *cache, dm_cblock_t cblock)
740{
741 if (test_and_clear_bit(from_cblock(cblock), cache->dirty_bitset)) {
742 if (atomic_dec_return(&cache->nr_dirty) == 0)
743 dm_table_event(cache->ti->table);
744 }
745
746 policy_clear_dirty(cache->policy, cblock);
747}
748
749/*----------------------------------------------------------------*/
750
751static bool block_size_is_power_of_two(struct cache *cache)
752{
753 return cache->sectors_per_block_shift >= 0;
754}
755
756/* gcc on ARM generates spurious references to __udivdi3 and __umoddi3 */
757#if defined(CONFIG_ARM) && __GNUC__ == 4 && __GNUC_MINOR__ <= 6
758__always_inline
759#endif
760static dm_block_t block_div(dm_block_t b, uint32_t n)
761{
762 do_div(b, n);
763
764 return b;
765}
766
767static dm_block_t oblocks_per_dblock(struct cache *cache)
768{
769 dm_block_t oblocks = cache->discard_block_size;
770
771 if (block_size_is_power_of_two(cache))
772 oblocks >>= cache->sectors_per_block_shift;
773 else
774 oblocks = block_div(oblocks, cache->sectors_per_block);
775
776 return oblocks;
777}
778
779static dm_dblock_t oblock_to_dblock(struct cache *cache, dm_oblock_t oblock)
780{
781 return to_dblock(block_div(from_oblock(oblock),
782 oblocks_per_dblock(cache)));
783}
784
785static void set_discard(struct cache *cache, dm_dblock_t b)
786{
787 unsigned long flags;
788
789 BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks));
790 atomic_inc(&cache->stats.discard_count);
791
792 spin_lock_irqsave(&cache->lock, flags);
793 set_bit(from_dblock(b), cache->discard_bitset);
794 spin_unlock_irqrestore(&cache->lock, flags);
795}
796
797static void clear_discard(struct cache *cache, dm_dblock_t b)
798{
799 unsigned long flags;
800
801 spin_lock_irqsave(&cache->lock, flags);
802 clear_bit(from_dblock(b), cache->discard_bitset);
803 spin_unlock_irqrestore(&cache->lock, flags);
804}
805
806static bool is_discarded(struct cache *cache, dm_dblock_t b)
807{
808 int r;
809 unsigned long flags;
810
811 spin_lock_irqsave(&cache->lock, flags);
812 r = test_bit(from_dblock(b), cache->discard_bitset);
813 spin_unlock_irqrestore(&cache->lock, flags);
814
815 return r;
816}
817
818static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
819{
820 int r;
821 unsigned long flags;
822
823 spin_lock_irqsave(&cache->lock, flags);
824 r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
825 cache->discard_bitset);
826 spin_unlock_irqrestore(&cache->lock, flags);
827
828 return r;
829}
830
831/*----------------------------------------------------------------
832 * Remapping
833 *--------------------------------------------------------------*/
834static void remap_to_origin(struct cache *cache, struct bio *bio)
835{
836 bio_set_dev(bio, cache->origin_dev->bdev);
837}
838
839static void remap_to_cache(struct cache *cache, struct bio *bio,
840 dm_cblock_t cblock)
841{
842 sector_t bi_sector = bio->bi_iter.bi_sector;
843 sector_t block = from_cblock(cblock);
844
845 bio_set_dev(bio, cache->cache_dev->bdev);
846 if (!block_size_is_power_of_two(cache))
847 bio->bi_iter.bi_sector =
848 (block * cache->sectors_per_block) +
849 sector_div(bi_sector, cache->sectors_per_block);
850 else
851 bio->bi_iter.bi_sector =
852 (block << cache->sectors_per_block_shift) |
853 (bi_sector & (cache->sectors_per_block - 1));
854}
855
856static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio)
857{
858 unsigned long flags;
859 size_t pb_data_size = get_per_bio_data_size(cache);
860 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
861
862 spin_lock_irqsave(&cache->lock, flags);
863 if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) &&
864 bio_op(bio) != REQ_OP_DISCARD) {
865 pb->tick = true;
866 cache->need_tick_bio = false;
867 }
868 spin_unlock_irqrestore(&cache->lock, flags);
869}
870
871static void remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
872 dm_oblock_t oblock)
873{
874 // FIXME: this is called way too much.
875 check_if_tick_bio_needed(cache, bio);
876 remap_to_origin(cache, bio);
877 if (bio_data_dir(bio) == WRITE)
878 clear_discard(cache, oblock_to_dblock(cache, oblock));
879}
880
881static void remap_to_cache_dirty(struct cache *cache, struct bio *bio,
882 dm_oblock_t oblock, dm_cblock_t cblock)
883{
884 check_if_tick_bio_needed(cache, bio);
885 remap_to_cache(cache, bio, cblock);
886 if (bio_data_dir(bio) == WRITE) {
887 set_dirty(cache, cblock);
888 clear_discard(cache, oblock_to_dblock(cache, oblock));
889 }
890}
891
892static dm_oblock_t get_bio_block(struct cache *cache, struct bio *bio)
893{
894 sector_t block_nr = bio->bi_iter.bi_sector;
895
896 if (!block_size_is_power_of_two(cache))
897 (void) sector_div(block_nr, cache->sectors_per_block);
898 else
899 block_nr >>= cache->sectors_per_block_shift;
900
901 return to_oblock(block_nr);
902}
903
904static bool accountable_bio(struct cache *cache, struct bio *bio)
905{
906 return bio_op(bio) != REQ_OP_DISCARD;
907}
908
909static void accounted_begin(struct cache *cache, struct bio *bio)
910{
911 size_t pb_data_size = get_per_bio_data_size(cache);
912 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
913
914 if (accountable_bio(cache, bio)) {
915 pb->len = bio_sectors(bio);
916 iot_io_begin(&cache->tracker, pb->len);
917 }
918}
919
920static void accounted_complete(struct cache *cache, struct bio *bio)
921{
922 size_t pb_data_size = get_per_bio_data_size(cache);
923 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
924
925 iot_io_end(&cache->tracker, pb->len);
926}
927
928static void accounted_request(struct cache *cache, struct bio *bio)
929{
930 accounted_begin(cache, bio);
931 generic_make_request(bio);
932}
933
934static void issue_op(struct bio *bio, void *context)
935{
936 struct cache *cache = context;
937 accounted_request(cache, bio);
938}
939
940static void defer_writethrough_bio(struct cache *cache, struct bio *bio)
941{
942 unsigned long flags;
943
944 spin_lock_irqsave(&cache->lock, flags);
945 bio_list_add(&cache->deferred_writethrough_bios, bio);
946 spin_unlock_irqrestore(&cache->lock, flags);
947
948 wake_deferred_writethrough_worker(cache);
949}
950
951static void writethrough_endio(struct bio *bio)
952{
953 struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT);
954
955 dm_unhook_bio(&pb->hook_info, bio);
956
957 if (bio->bi_status) {
958 bio_endio(bio);
959 return;
960 }
961
962 dm_bio_restore(&pb->bio_details, bio);
963 remap_to_cache(pb->cache, bio, pb->cblock);
964
965 /*
966 * We can't issue this bio directly, since we're in interrupt
967 * context. So it gets put on a bio list for processing by the
968 * worker thread.
969 */
970 defer_writethrough_bio(pb->cache, bio);
971}
972
973/*
974 * FIXME: send in parallel, huge latency as is.
975 * When running in writethrough mode we need to send writes to clean blocks
976 * to both the cache and origin devices. In future we'd like to clone the
977 * bio and send them in parallel, but for now we're doing them in
978 * series as this is easier.
979 */
980static void remap_to_origin_then_cache(struct cache *cache, struct bio *bio,
981 dm_oblock_t oblock, dm_cblock_t cblock)
982{
983 struct per_bio_data *pb = get_per_bio_data(bio, PB_DATA_SIZE_WT);
984
985 pb->cache = cache;
986 pb->cblock = cblock;
987 dm_hook_bio(&pb->hook_info, bio, writethrough_endio, NULL);
988 dm_bio_record(&pb->bio_details, bio);
989
990 remap_to_origin_clear_discard(pb->cache, bio, oblock);
991}
992
993/*----------------------------------------------------------------
994 * Failure modes
995 *--------------------------------------------------------------*/
996static enum cache_metadata_mode get_cache_mode(struct cache *cache)
997{
998 return cache->features.mode;
999}
1000
1001static const char *cache_device_name(struct cache *cache)
1002{
1003 return dm_device_name(dm_table_get_md(cache->ti->table));
1004}
1005
1006static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode)
1007{
1008 const char *descs[] = {
1009 "write",
1010 "read-only",
1011 "fail"
1012 };
1013
1014 dm_table_event(cache->ti->table);
1015 DMINFO("%s: switching cache to %s mode",
1016 cache_device_name(cache), descs[(int)mode]);
1017}
1018
1019static void set_cache_mode(struct cache *cache, enum cache_metadata_mode new_mode)
1020{
1021 bool needs_check;
1022 enum cache_metadata_mode old_mode = get_cache_mode(cache);
1023
1024 if (dm_cache_metadata_needs_check(cache->cmd, &needs_check)) {
1025 DMERR("%s: unable to read needs_check flag, setting failure mode.",
1026 cache_device_name(cache));
1027 new_mode = CM_FAIL;
1028 }
1029
1030 if (new_mode == CM_WRITE && needs_check) {
1031 DMERR("%s: unable to switch cache to write mode until repaired.",
1032 cache_device_name(cache));
1033 if (old_mode != new_mode)
1034 new_mode = old_mode;
1035 else
1036 new_mode = CM_READ_ONLY;
1037 }
1038
1039 /* Never move out of fail mode */
1040 if (old_mode == CM_FAIL)
1041 new_mode = CM_FAIL;
1042
1043 switch (new_mode) {
1044 case CM_FAIL:
1045 case CM_READ_ONLY:
1046 dm_cache_metadata_set_read_only(cache->cmd);
1047 break;
1048
1049 case CM_WRITE:
1050 dm_cache_metadata_set_read_write(cache->cmd);
1051 break;
1052 }
1053
1054 cache->features.mode = new_mode;
1055
1056 if (new_mode != old_mode)
1057 notify_mode_switch(cache, new_mode);
1058}
1059
1060static void abort_transaction(struct cache *cache)
1061{
1062 const char *dev_name = cache_device_name(cache);
1063
1064 if (get_cache_mode(cache) >= CM_READ_ONLY)
1065 return;
1066
1067 if (dm_cache_metadata_set_needs_check(cache->cmd)) {
1068 DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
1069 set_cache_mode(cache, CM_FAIL);
1070 }
1071
1072 DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
1073 if (dm_cache_metadata_abort(cache->cmd)) {
1074 DMERR("%s: failed to abort metadata transaction", dev_name);
1075 set_cache_mode(cache, CM_FAIL);
1076 }
1077}
1078
1079static void metadata_operation_failed(struct cache *cache, const char *op, int r)
1080{
1081 DMERR_LIMIT("%s: metadata operation '%s' failed: error = %d",
1082 cache_device_name(cache), op, r);
1083 abort_transaction(cache);
1084 set_cache_mode(cache, CM_READ_ONLY);
1085}
1086
1087/*----------------------------------------------------------------*/
1088
1089static void load_stats(struct cache *cache)
1090{
1091 struct dm_cache_statistics stats;
1092
1093 dm_cache_metadata_get_stats(cache->cmd, &stats);
1094 atomic_set(&cache->stats.read_hit, stats.read_hits);
1095 atomic_set(&cache->stats.read_miss, stats.read_misses);
1096 atomic_set(&cache->stats.write_hit, stats.write_hits);
1097 atomic_set(&cache->stats.write_miss, stats.write_misses);
1098}
1099
1100static void save_stats(struct cache *cache)
1101{
1102 struct dm_cache_statistics stats;
1103
1104 if (get_cache_mode(cache) >= CM_READ_ONLY)
1105 return;
1106
1107 stats.read_hits = atomic_read(&cache->stats.read_hit);
1108 stats.read_misses = atomic_read(&cache->stats.read_miss);
1109 stats.write_hits = atomic_read(&cache->stats.write_hit);
1110 stats.write_misses = atomic_read(&cache->stats.write_miss);
1111
1112 dm_cache_metadata_set_stats(cache->cmd, &stats);
1113}
1114
1115static void update_stats(struct cache_stats *stats, enum policy_operation op)
1116{
1117 switch (op) {
1118 case POLICY_PROMOTE:
1119 atomic_inc(&stats->promotion);
1120 break;
1121
1122 case POLICY_DEMOTE:
1123 atomic_inc(&stats->demotion);
1124 break;
1125
1126 case POLICY_WRITEBACK:
1127 atomic_inc(&stats->writeback);
1128 break;
1129 }
1130}
1131
1132/*----------------------------------------------------------------
1133 * Migration processing
1134 *
1135 * Migration covers moving data from the origin device to the cache, or
1136 * vice versa.
1137 *--------------------------------------------------------------*/
1138
1139static void inc_io_migrations(struct cache *cache)
1140{
1141 atomic_inc(&cache->nr_io_migrations);
1142}
1143
1144static void dec_io_migrations(struct cache *cache)
1145{
1146 atomic_dec(&cache->nr_io_migrations);
1147}
1148
1149static bool discard_or_flush(struct bio *bio)
1150{
1151 return bio_op(bio) == REQ_OP_DISCARD || op_is_flush(bio->bi_opf);
1152}
1153
1154static void calc_discard_block_range(struct cache *cache, struct bio *bio,
1155 dm_dblock_t *b, dm_dblock_t *e)
1156{
1157 sector_t sb = bio->bi_iter.bi_sector;
1158 sector_t se = bio_end_sector(bio);
1159
1160 *b = to_dblock(dm_sector_div_up(sb, cache->discard_block_size));
1161
1162 if (se - sb < cache->discard_block_size)
1163 *e = *b;
1164 else
1165 *e = to_dblock(block_div(se, cache->discard_block_size));
1166}
1167
1168/*----------------------------------------------------------------*/
1169
1170static void prevent_background_work(struct cache *cache)
1171{
1172 lockdep_off();
1173 down_write(&cache->background_work_lock);
1174 lockdep_on();
1175}
1176
1177static void allow_background_work(struct cache *cache)
1178{
1179 lockdep_off();
1180 up_write(&cache->background_work_lock);
1181 lockdep_on();
1182}
1183
1184static bool background_work_begin(struct cache *cache)
1185{
1186 bool r;
1187
1188 lockdep_off();
1189 r = down_read_trylock(&cache->background_work_lock);
1190 lockdep_on();
1191
1192 return r;
1193}
1194
1195static void background_work_end(struct cache *cache)
1196{
1197 lockdep_off();
1198 up_read(&cache->background_work_lock);
1199 lockdep_on();
1200}
1201
1202/*----------------------------------------------------------------*/
1203
1204static bool bio_writes_complete_block(struct cache *cache, struct bio *bio)
1205{
1206 return (bio_data_dir(bio) == WRITE) &&
1207 (bio->bi_iter.bi_size == (cache->sectors_per_block << SECTOR_SHIFT));
1208}
1209
1210static bool optimisable_bio(struct cache *cache, struct bio *bio, dm_oblock_t block)
1211{
1212 return writeback_mode(&cache->features) &&
1213 (is_discarded_oblock(cache, block) || bio_writes_complete_block(cache, bio));
1214}
1215
1216static void quiesce(struct dm_cache_migration *mg,
1217 void (*continuation)(struct work_struct *))
1218{
1219 init_continuation(&mg->k, continuation);
1220 dm_cell_quiesce_v2(mg->cache->prison, mg->cell, &mg->k.ws);
1221}
1222
1223static struct dm_cache_migration *ws_to_mg(struct work_struct *ws)
1224{
1225 struct continuation *k = container_of(ws, struct continuation, ws);
1226 return container_of(k, struct dm_cache_migration, k);
1227}
1228
1229static void copy_complete(int read_err, unsigned long write_err, void *context)
1230{
1231 struct dm_cache_migration *mg = container_of(context, struct dm_cache_migration, k);
1232
1233 if (read_err || write_err)
1234 mg->k.input = BLK_STS_IOERR;
1235
1236 queue_continuation(mg->cache->wq, &mg->k);
1237}
1238
1239static int copy(struct dm_cache_migration *mg, bool promote)
1240{
1241 int r;
1242 struct dm_io_region o_region, c_region;
1243 struct cache *cache = mg->cache;
1244
1245 o_region.bdev = cache->origin_dev->bdev;
1246 o_region.sector = from_oblock(mg->op->oblock) * cache->sectors_per_block;
1247 o_region.count = cache->sectors_per_block;
1248
1249 c_region.bdev = cache->cache_dev->bdev;
1250 c_region.sector = from_cblock(mg->op->cblock) * cache->sectors_per_block;
1251 c_region.count = cache->sectors_per_block;
1252
1253 if (promote)
1254 r = dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k);
1255 else
1256 r = dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k);
1257
1258 return r;
1259}
1260
1261static void bio_drop_shared_lock(struct cache *cache, struct bio *bio)
1262{
1263 size_t pb_data_size = get_per_bio_data_size(cache);
1264 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
1265
1266 if (pb->cell && dm_cell_put_v2(cache->prison, pb->cell))
1267 free_prison_cell(cache, pb->cell);
1268 pb->cell = NULL;
1269}
1270
1271static void overwrite_endio(struct bio *bio)
1272{
1273 struct dm_cache_migration *mg = bio->bi_private;
1274 struct cache *cache = mg->cache;
1275 size_t pb_data_size = get_per_bio_data_size(cache);
1276 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
1277
1278 dm_unhook_bio(&pb->hook_info, bio);
1279
1280 if (bio->bi_status)
1281 mg->k.input = bio->bi_status;
1282
1283 queue_continuation(mg->cache->wq, &mg->k);
1284}
1285
1286static void overwrite(struct dm_cache_migration *mg,
1287 void (*continuation)(struct work_struct *))
1288{
1289 struct bio *bio = mg->overwrite_bio;
1290 size_t pb_data_size = get_per_bio_data_size(mg->cache);
1291 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
1292
1293 dm_hook_bio(&pb->hook_info, bio, overwrite_endio, mg);
1294
1295 /*
1296 * The overwrite bio is part of the copy operation, as such it does
1297 * not set/clear discard or dirty flags.
1298 */
1299 if (mg->op->op == POLICY_PROMOTE)
1300 remap_to_cache(mg->cache, bio, mg->op->cblock);
1301 else
1302 remap_to_origin(mg->cache, bio);
1303
1304 init_continuation(&mg->k, continuation);
1305 accounted_request(mg->cache, bio);
1306}
1307
1308/*
1309 * Migration steps:
1310 *
1311 * 1) exclusive lock preventing WRITEs
1312 * 2) quiesce
1313 * 3) copy or issue overwrite bio
1314 * 4) upgrade to exclusive lock preventing READs and WRITEs
1315 * 5) quiesce
1316 * 6) update metadata and commit
1317 * 7) unlock
1318 */
1319static void mg_complete(struct dm_cache_migration *mg, bool success)
1320{
1321 struct bio_list bios;
1322 struct cache *cache = mg->cache;
1323 struct policy_work *op = mg->op;
1324 dm_cblock_t cblock = op->cblock;
1325
1326 if (success)
1327 update_stats(&cache->stats, op->op);
1328
1329 switch (op->op) {
1330 case POLICY_PROMOTE:
1331 clear_discard(cache, oblock_to_dblock(cache, op->oblock));
1332 policy_complete_background_work(cache->policy, op, success);
1333
1334 if (mg->overwrite_bio) {
1335 if (success)
1336 force_set_dirty(cache, cblock);
1337 else if (mg->k.input)
1338 mg->overwrite_bio->bi_status = mg->k.input;
1339 else
1340 mg->overwrite_bio->bi_status = BLK_STS_IOERR;
1341 bio_endio(mg->overwrite_bio);
1342 } else {
1343 if (success)
1344 force_clear_dirty(cache, cblock);
1345 dec_io_migrations(cache);
1346 }
1347 break;
1348
1349 case POLICY_DEMOTE:
1350 /*
1351 * We clear dirty here to update the nr_dirty counter.
1352 */
1353 if (success)
1354 force_clear_dirty(cache, cblock);
1355 policy_complete_background_work(cache->policy, op, success);
1356 dec_io_migrations(cache);
1357 break;
1358
1359 case POLICY_WRITEBACK:
1360 if (success)
1361 force_clear_dirty(cache, cblock);
1362 policy_complete_background_work(cache->policy, op, success);
1363 dec_io_migrations(cache);
1364 break;
1365 }
1366
1367 bio_list_init(&bios);
1368 if (mg->cell) {
1369 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1370 free_prison_cell(cache, mg->cell);
1371 }
1372
1373 free_migration(mg);
1374 defer_bios(cache, &bios);
1375 wake_migration_worker(cache);
1376
1377 background_work_end(cache);
1378}
1379
1380static void mg_success(struct work_struct *ws)
1381{
1382 struct dm_cache_migration *mg = ws_to_mg(ws);
1383 mg_complete(mg, mg->k.input == 0);
1384}
1385
1386static void mg_update_metadata(struct work_struct *ws)
1387{
1388 int r;
1389 struct dm_cache_migration *mg = ws_to_mg(ws);
1390 struct cache *cache = mg->cache;
1391 struct policy_work *op = mg->op;
1392
1393 switch (op->op) {
1394 case POLICY_PROMOTE:
1395 r = dm_cache_insert_mapping(cache->cmd, op->cblock, op->oblock);
1396 if (r) {
1397 DMERR_LIMIT("%s: migration failed; couldn't insert mapping",
1398 cache_device_name(cache));
1399 metadata_operation_failed(cache, "dm_cache_insert_mapping", r);
1400
1401 mg_complete(mg, false);
1402 return;
1403 }
1404 mg_complete(mg, true);
1405 break;
1406
1407 case POLICY_DEMOTE:
1408 r = dm_cache_remove_mapping(cache->cmd, op->cblock);
1409 if (r) {
1410 DMERR_LIMIT("%s: migration failed; couldn't update on disk metadata",
1411 cache_device_name(cache));
1412 metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1413
1414 mg_complete(mg, false);
1415 return;
1416 }
1417
1418 /*
1419 * It would be nice if we only had to commit when a REQ_FLUSH
1420 * comes through. But there's one scenario that we have to
1421 * look out for:
1422 *
1423 * - vblock x in a cache block
1424 * - domotion occurs
1425 * - cache block gets reallocated and over written
1426 * - crash
1427 *
1428 * When we recover, because there was no commit the cache will
1429 * rollback to having the data for vblock x in the cache block.
1430 * But the cache block has since been overwritten, so it'll end
1431 * up pointing to data that was never in 'x' during the history
1432 * of the device.
1433 *
1434 * To avoid this issue we require a commit as part of the
1435 * demotion operation.
1436 */
1437 init_continuation(&mg->k, mg_success);
1438 continue_after_commit(&cache->committer, &mg->k);
1439 schedule_commit(&cache->committer);
1440 break;
1441
1442 case POLICY_WRITEBACK:
1443 mg_complete(mg, true);
1444 break;
1445 }
1446}
1447
1448static void mg_update_metadata_after_copy(struct work_struct *ws)
1449{
1450 struct dm_cache_migration *mg = ws_to_mg(ws);
1451
1452 /*
1453 * Did the copy succeed?
1454 */
1455 if (mg->k.input)
1456 mg_complete(mg, false);
1457 else
1458 mg_update_metadata(ws);
1459}
1460
1461static void mg_upgrade_lock(struct work_struct *ws)
1462{
1463 int r;
1464 struct dm_cache_migration *mg = ws_to_mg(ws);
1465
1466 /*
1467 * Did the copy succeed?
1468 */
1469 if (mg->k.input)
1470 mg_complete(mg, false);
1471
1472 else {
1473 /*
1474 * Now we want the lock to prevent both reads and writes.
1475 */
1476 r = dm_cell_lock_promote_v2(mg->cache->prison, mg->cell,
1477 READ_WRITE_LOCK_LEVEL);
1478 if (r < 0)
1479 mg_complete(mg, false);
1480
1481 else if (r)
1482 quiesce(mg, mg_update_metadata);
1483
1484 else
1485 mg_update_metadata(ws);
1486 }
1487}
1488
1489static void mg_full_copy(struct work_struct *ws)
1490{
1491 struct dm_cache_migration *mg = ws_to_mg(ws);
1492 struct cache *cache = mg->cache;
1493 struct policy_work *op = mg->op;
1494 bool is_policy_promote = (op->op == POLICY_PROMOTE);
1495
1496 if ((!is_policy_promote && !is_dirty(cache, op->cblock)) ||
1497 is_discarded_oblock(cache, op->oblock)) {
1498 mg_upgrade_lock(ws);
1499 return;
1500 }
1501
1502 init_continuation(&mg->k, mg_upgrade_lock);
1503
1504 if (copy(mg, is_policy_promote)) {
1505 DMERR_LIMIT("%s: migration copy failed", cache_device_name(cache));
1506 mg->k.input = BLK_STS_IOERR;
1507 mg_complete(mg, false);
1508 }
1509}
1510
1511static void mg_copy(struct work_struct *ws)
1512{
1513 struct dm_cache_migration *mg = ws_to_mg(ws);
1514
1515 if (mg->overwrite_bio) {
1516 /*
1517 * No exclusive lock was held when we last checked if the bio
1518 * was optimisable. So we have to check again in case things
1519 * have changed (eg, the block may no longer be discarded).
1520 */
1521 if (!optimisable_bio(mg->cache, mg->overwrite_bio, mg->op->oblock)) {
1522 /*
1523 * Fallback to a real full copy after doing some tidying up.
1524 */
1525 bool rb = bio_detain_shared(mg->cache, mg->op->oblock, mg->overwrite_bio);
1526 BUG_ON(rb); /* An exclussive lock must _not_ be held for this block */
1527 mg->overwrite_bio = NULL;
1528 inc_io_migrations(mg->cache);
1529 mg_full_copy(ws);
1530 return;
1531 }
1532
1533 /*
1534 * It's safe to do this here, even though it's new data
1535 * because all IO has been locked out of the block.
1536 *
1537 * mg_lock_writes() already took READ_WRITE_LOCK_LEVEL
1538 * so _not_ using mg_upgrade_lock() as continutation.
1539 */
1540 overwrite(mg, mg_update_metadata_after_copy);
1541
1542 } else
1543 mg_full_copy(ws);
1544}
1545
1546static int mg_lock_writes(struct dm_cache_migration *mg)
1547{
1548 int r;
1549 struct dm_cell_key_v2 key;
1550 struct cache *cache = mg->cache;
1551 struct dm_bio_prison_cell_v2 *prealloc;
1552
1553 prealloc = alloc_prison_cell(cache);
1554 if (!prealloc) {
1555 DMERR_LIMIT("%s: alloc_prison_cell failed", cache_device_name(cache));
1556 mg_complete(mg, false);
1557 return -ENOMEM;
1558 }
1559
1560 /*
1561 * Prevent writes to the block, but allow reads to continue.
1562 * Unless we're using an overwrite bio, in which case we lock
1563 * everything.
1564 */
1565 build_key(mg->op->oblock, oblock_succ(mg->op->oblock), &key);
1566 r = dm_cell_lock_v2(cache->prison, &key,
1567 mg->overwrite_bio ? READ_WRITE_LOCK_LEVEL : WRITE_LOCK_LEVEL,
1568 prealloc, &mg->cell);
1569 if (r < 0) {
1570 free_prison_cell(cache, prealloc);
1571 mg_complete(mg, false);
1572 return r;
1573 }
1574
1575 if (mg->cell != prealloc)
1576 free_prison_cell(cache, prealloc);
1577
1578 if (r == 0)
1579 mg_copy(&mg->k.ws);
1580 else
1581 quiesce(mg, mg_copy);
1582
1583 return 0;
1584}
1585
1586static int mg_start(struct cache *cache, struct policy_work *op, struct bio *bio)
1587{
1588 struct dm_cache_migration *mg;
1589
1590 if (!background_work_begin(cache)) {
1591 policy_complete_background_work(cache->policy, op, false);
1592 return -EPERM;
1593 }
1594
1595 mg = alloc_migration(cache);
1596 if (!mg) {
1597 policy_complete_background_work(cache->policy, op, false);
1598 background_work_end(cache);
1599 return -ENOMEM;
1600 }
1601
1602 memset(mg, 0, sizeof(*mg));
1603
1604 mg->cache = cache;
1605 mg->op = op;
1606 mg->overwrite_bio = bio;
1607
1608 if (!bio)
1609 inc_io_migrations(cache);
1610
1611 return mg_lock_writes(mg);
1612}
1613
1614/*----------------------------------------------------------------
1615 * invalidation processing
1616 *--------------------------------------------------------------*/
1617
1618static void invalidate_complete(struct dm_cache_migration *mg, bool success)
1619{
1620 struct bio_list bios;
1621 struct cache *cache = mg->cache;
1622
1623 bio_list_init(&bios);
1624 if (dm_cell_unlock_v2(cache->prison, mg->cell, &bios))
1625 free_prison_cell(cache, mg->cell);
1626
1627 if (!success && mg->overwrite_bio)
1628 bio_io_error(mg->overwrite_bio);
1629
1630 free_migration(mg);
1631 defer_bios(cache, &bios);
1632
1633 background_work_end(cache);
1634}
1635
1636static void invalidate_completed(struct work_struct *ws)
1637{
1638 struct dm_cache_migration *mg = ws_to_mg(ws);
1639 invalidate_complete(mg, !mg->k.input);
1640}
1641
1642static int invalidate_cblock(struct cache *cache, dm_cblock_t cblock)
1643{
1644 int r = policy_invalidate_mapping(cache->policy, cblock);
1645 if (!r) {
1646 r = dm_cache_remove_mapping(cache->cmd, cblock);
1647 if (r) {
1648 DMERR_LIMIT("%s: invalidation failed; couldn't update on disk metadata",
1649 cache_device_name(cache));
1650 metadata_operation_failed(cache, "dm_cache_remove_mapping", r);
1651 }
1652
1653 } else if (r == -ENODATA) {
1654 /*
1655 * Harmless, already unmapped.
1656 */
1657 r = 0;
1658
1659 } else
1660 DMERR("%s: policy_invalidate_mapping failed", cache_device_name(cache));
1661
1662 return r;
1663}
1664
1665static void invalidate_remove(struct work_struct *ws)
1666{
1667 int r;
1668 struct dm_cache_migration *mg = ws_to_mg(ws);
1669 struct cache *cache = mg->cache;
1670
1671 r = invalidate_cblock(cache, mg->invalidate_cblock);
1672 if (r) {
1673 invalidate_complete(mg, false);
1674 return;
1675 }
1676
1677 init_continuation(&mg->k, invalidate_completed);
1678 continue_after_commit(&cache->committer, &mg->k);
1679 remap_to_origin_clear_discard(cache, mg->overwrite_bio, mg->invalidate_oblock);
1680 mg->overwrite_bio = NULL;
1681 schedule_commit(&cache->committer);
1682}
1683
1684static int invalidate_lock(struct dm_cache_migration *mg)
1685{
1686 int r;
1687 struct dm_cell_key_v2 key;
1688 struct cache *cache = mg->cache;
1689 struct dm_bio_prison_cell_v2 *prealloc;
1690
1691 prealloc = alloc_prison_cell(cache);
1692 if (!prealloc) {
1693 invalidate_complete(mg, false);
1694 return -ENOMEM;
1695 }
1696
1697 build_key(mg->invalidate_oblock, oblock_succ(mg->invalidate_oblock), &key);
1698 r = dm_cell_lock_v2(cache->prison, &key,
1699 READ_WRITE_LOCK_LEVEL, prealloc, &mg->cell);
1700 if (r < 0) {
1701 free_prison_cell(cache, prealloc);
1702 invalidate_complete(mg, false);
1703 return r;
1704 }
1705
1706 if (mg->cell != prealloc)
1707 free_prison_cell(cache, prealloc);
1708
1709 if (r)
1710 quiesce(mg, invalidate_remove);
1711
1712 else {
1713 /*
1714 * We can't call invalidate_remove() directly here because we
1715 * might still be in request context.
1716 */
1717 init_continuation(&mg->k, invalidate_remove);
1718 queue_work(cache->wq, &mg->k.ws);
1719 }
1720
1721 return 0;
1722}
1723
1724static int invalidate_start(struct cache *cache, dm_cblock_t cblock,
1725 dm_oblock_t oblock, struct bio *bio)
1726{
1727 struct dm_cache_migration *mg;
1728
1729 if (!background_work_begin(cache))
1730 return -EPERM;
1731
1732 mg = alloc_migration(cache);
1733 if (!mg) {
1734 background_work_end(cache);
1735 return -ENOMEM;
1736 }
1737
1738 memset(mg, 0, sizeof(*mg));
1739
1740 mg->cache = cache;
1741 mg->overwrite_bio = bio;
1742 mg->invalidate_cblock = cblock;
1743 mg->invalidate_oblock = oblock;
1744
1745 return invalidate_lock(mg);
1746}
1747
1748/*----------------------------------------------------------------
1749 * bio processing
1750 *--------------------------------------------------------------*/
1751
1752enum busy {
1753 IDLE,
1754 BUSY
1755};
1756
1757static enum busy spare_migration_bandwidth(struct cache *cache)
1758{
1759 bool idle = iot_idle_for(&cache->tracker, HZ);
1760 sector_t current_volume = (atomic_read(&cache->nr_io_migrations) + 1) *
1761 cache->sectors_per_block;
1762
1763 if (idle && current_volume <= cache->migration_threshold)
1764 return IDLE;
1765 else
1766 return BUSY;
1767}
1768
1769static void inc_hit_counter(struct cache *cache, struct bio *bio)
1770{
1771 atomic_inc(bio_data_dir(bio) == READ ?
1772 &cache->stats.read_hit : &cache->stats.write_hit);
1773}
1774
1775static void inc_miss_counter(struct cache *cache, struct bio *bio)
1776{
1777 atomic_inc(bio_data_dir(bio) == READ ?
1778 &cache->stats.read_miss : &cache->stats.write_miss);
1779}
1780
1781/*----------------------------------------------------------------*/
1782
1783static int map_bio(struct cache *cache, struct bio *bio, dm_oblock_t block,
1784 bool *commit_needed)
1785{
1786 int r, data_dir;
1787 bool rb, background_queued;
1788 dm_cblock_t cblock;
1789 size_t pb_data_size = get_per_bio_data_size(cache);
1790 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
1791
1792 *commit_needed = false;
1793
1794 rb = bio_detain_shared(cache, block, bio);
1795 if (!rb) {
1796 /*
1797 * An exclusive lock is held for this block, so we have to
1798 * wait. We set the commit_needed flag so the current
1799 * transaction will be committed asap, allowing this lock
1800 * to be dropped.
1801 */
1802 *commit_needed = true;
1803 return DM_MAPIO_SUBMITTED;
1804 }
1805
1806 data_dir = bio_data_dir(bio);
1807
1808 if (optimisable_bio(cache, bio, block)) {
1809 struct policy_work *op = NULL;
1810
1811 r = policy_lookup_with_work(cache->policy, block, &cblock, data_dir, true, &op);
1812 if (unlikely(r && r != -ENOENT)) {
1813 DMERR_LIMIT("%s: policy_lookup_with_work() failed with r = %d",
1814 cache_device_name(cache), r);
1815 bio_io_error(bio);
1816 return DM_MAPIO_SUBMITTED;
1817 }
1818
1819 if (r == -ENOENT && op) {
1820 bio_drop_shared_lock(cache, bio);
1821 BUG_ON(op->op != POLICY_PROMOTE);
1822 mg_start(cache, op, bio);
1823 return DM_MAPIO_SUBMITTED;
1824 }
1825 } else {
1826 r = policy_lookup(cache->policy, block, &cblock, data_dir, false, &background_queued);
1827 if (unlikely(r && r != -ENOENT)) {
1828 DMERR_LIMIT("%s: policy_lookup() failed with r = %d",
1829 cache_device_name(cache), r);
1830 bio_io_error(bio);
1831 return DM_MAPIO_SUBMITTED;
1832 }
1833
1834 if (background_queued)
1835 wake_migration_worker(cache);
1836 }
1837
1838 if (r == -ENOENT) {
1839 /*
1840 * Miss.
1841 */
1842 inc_miss_counter(cache, bio);
1843 if (pb->req_nr == 0) {
1844 accounted_begin(cache, bio);
1845 remap_to_origin_clear_discard(cache, bio, block);
1846
1847 } else {
1848 /*
1849 * This is a duplicate writethrough io that is no
1850 * longer needed because the block has been demoted.
1851 */
1852 bio_endio(bio);
1853 return DM_MAPIO_SUBMITTED;
1854 }
1855 } else {
1856 /*
1857 * Hit.
1858 */
1859 inc_hit_counter(cache, bio);
1860
1861 /*
1862 * Passthrough always maps to the origin, invalidating any
1863 * cache blocks that are written to.
1864 */
1865 if (passthrough_mode(&cache->features)) {
1866 if (bio_data_dir(bio) == WRITE) {
1867 bio_drop_shared_lock(cache, bio);
1868 atomic_inc(&cache->stats.demotion);
1869 invalidate_start(cache, cblock, block, bio);
1870 } else
1871 remap_to_origin_clear_discard(cache, bio, block);
1872
1873 } else {
1874 if (bio_data_dir(bio) == WRITE && writethrough_mode(&cache->features) &&
1875 !is_dirty(cache, cblock)) {
1876 remap_to_origin_then_cache(cache, bio, block, cblock);
1877 accounted_begin(cache, bio);
1878 } else
1879 remap_to_cache_dirty(cache, bio, block, cblock);
1880 }
1881 }
1882
1883 /*
1884 * dm core turns FUA requests into a separate payload and FLUSH req.
1885 */
1886 if (bio->bi_opf & REQ_FUA) {
1887 /*
1888 * issue_after_commit will call accounted_begin a second time. So
1889 * we call accounted_complete() to avoid double accounting.
1890 */
1891 accounted_complete(cache, bio);
1892 issue_after_commit(&cache->committer, bio);
1893 *commit_needed = true;
1894 return DM_MAPIO_SUBMITTED;
1895 }
1896
1897 return DM_MAPIO_REMAPPED;
1898}
1899
1900static bool process_bio(struct cache *cache, struct bio *bio)
1901{
1902 bool commit_needed;
1903
1904 if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED)
1905 generic_make_request(bio);
1906
1907 return commit_needed;
1908}
1909
1910/*
1911 * A non-zero return indicates read_only or fail_io mode.
1912 */
1913static int commit(struct cache *cache, bool clean_shutdown)
1914{
1915 int r;
1916
1917 if (get_cache_mode(cache) >= CM_READ_ONLY)
1918 return -EINVAL;
1919
1920 atomic_inc(&cache->stats.commit_count);
1921 r = dm_cache_commit(cache->cmd, clean_shutdown);
1922 if (r)
1923 metadata_operation_failed(cache, "dm_cache_commit", r);
1924
1925 return r;
1926}
1927
1928/*
1929 * Used by the batcher.
1930 */
1931static blk_status_t commit_op(void *context)
1932{
1933 struct cache *cache = context;
1934
1935 if (dm_cache_changed_this_transaction(cache->cmd))
1936 return errno_to_blk_status(commit(cache, false));
1937
1938 return 0;
1939}
1940
1941/*----------------------------------------------------------------*/
1942
1943static bool process_flush_bio(struct cache *cache, struct bio *bio)
1944{
1945 size_t pb_data_size = get_per_bio_data_size(cache);
1946 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
1947
1948 if (!pb->req_nr)
1949 remap_to_origin(cache, bio);
1950 else
1951 remap_to_cache(cache, bio, 0);
1952
1953 issue_after_commit(&cache->committer, bio);
1954 return true;
1955}
1956
1957static bool process_discard_bio(struct cache *cache, struct bio *bio)
1958{
1959 dm_dblock_t b, e;
1960
1961 // FIXME: do we need to lock the region? Or can we just assume the
1962 // user wont be so foolish as to issue discard concurrently with
1963 // other IO?
1964 calc_discard_block_range(cache, bio, &b, &e);
1965 while (b != e) {
1966 set_discard(cache, b);
1967 b = to_dblock(from_dblock(b) + 1);
1968 }
1969
1970 bio_endio(bio);
1971
1972 return false;
1973}
1974
1975static void process_deferred_bios(struct work_struct *ws)
1976{
1977 struct cache *cache = container_of(ws, struct cache, deferred_bio_worker);
1978
1979 unsigned long flags;
1980 bool commit_needed = false;
1981 struct bio_list bios;
1982 struct bio *bio;
1983
1984 bio_list_init(&bios);
1985
1986 spin_lock_irqsave(&cache->lock, flags);
1987 bio_list_merge(&bios, &cache->deferred_bios);
1988 bio_list_init(&cache->deferred_bios);
1989 spin_unlock_irqrestore(&cache->lock, flags);
1990
1991 while ((bio = bio_list_pop(&bios))) {
1992 if (bio->bi_opf & REQ_PREFLUSH)
1993 commit_needed = process_flush_bio(cache, bio) || commit_needed;
1994
1995 else if (bio_op(bio) == REQ_OP_DISCARD)
1996 commit_needed = process_discard_bio(cache, bio) || commit_needed;
1997
1998 else
1999 commit_needed = process_bio(cache, bio) || commit_needed;
2000 }
2001
2002 if (commit_needed)
2003 schedule_commit(&cache->committer);
2004}
2005
2006static void process_deferred_writethrough_bios(struct work_struct *ws)
2007{
2008 struct cache *cache = container_of(ws, struct cache, deferred_writethrough_worker);
2009
2010 unsigned long flags;
2011 struct bio_list bios;
2012 struct bio *bio;
2013
2014 bio_list_init(&bios);
2015
2016 spin_lock_irqsave(&cache->lock, flags);
2017 bio_list_merge(&bios, &cache->deferred_writethrough_bios);
2018 bio_list_init(&cache->deferred_writethrough_bios);
2019 spin_unlock_irqrestore(&cache->lock, flags);
2020
2021 /*
2022 * These bios have already been through accounted_begin()
2023 */
2024 while ((bio = bio_list_pop(&bios)))
2025 generic_make_request(bio);
2026}
2027
2028/*----------------------------------------------------------------
2029 * Main worker loop
2030 *--------------------------------------------------------------*/
2031
2032static void requeue_deferred_bios(struct cache *cache)
2033{
2034 struct bio *bio;
2035 struct bio_list bios;
2036
2037 bio_list_init(&bios);
2038 bio_list_merge(&bios, &cache->deferred_bios);
2039 bio_list_init(&cache->deferred_bios);
2040
2041 while ((bio = bio_list_pop(&bios))) {
2042 bio->bi_status = BLK_STS_DM_REQUEUE;
2043 bio_endio(bio);
2044 }
2045}
2046
2047/*
2048 * We want to commit periodically so that not too much
2049 * unwritten metadata builds up.
2050 */
2051static void do_waker(struct work_struct *ws)
2052{
2053 struct cache *cache = container_of(to_delayed_work(ws), struct cache, waker);
2054
2055 policy_tick(cache->policy, true);
2056 wake_migration_worker(cache);
2057 schedule_commit(&cache->committer);
2058 queue_delayed_work(cache->wq, &cache->waker, COMMIT_PERIOD);
2059}
2060
2061static void check_migrations(struct work_struct *ws)
2062{
2063 int r;
2064 struct policy_work *op;
2065 struct cache *cache = container_of(ws, struct cache, migration_worker);
2066 enum busy b;
2067
2068 for (;;) {
2069 b = spare_migration_bandwidth(cache);
2070
2071 r = policy_get_background_work(cache->policy, b == IDLE, &op);
2072 if (r == -ENODATA)
2073 break;
2074
2075 if (r) {
2076 DMERR_LIMIT("%s: policy_background_work failed",
2077 cache_device_name(cache));
2078 break;
2079 }
2080
2081 r = mg_start(cache, op, NULL);
2082 if (r)
2083 break;
2084 }
2085}
2086
2087/*----------------------------------------------------------------
2088 * Target methods
2089 *--------------------------------------------------------------*/
2090
2091/*
2092 * This function gets called on the error paths of the constructor, so we
2093 * have to cope with a partially initialised struct.
2094 */
2095static void destroy(struct cache *cache)
2096{
2097 unsigned i;
2098
2099 mempool_destroy(cache->migration_pool);
2100
2101 if (cache->prison)
2102 dm_bio_prison_destroy_v2(cache->prison);
2103
2104 if (cache->wq)
2105 destroy_workqueue(cache->wq);
2106
2107 if (cache->dirty_bitset)
2108 free_bitset(cache->dirty_bitset);
2109
2110 if (cache->discard_bitset)
2111 free_bitset(cache->discard_bitset);
2112
2113 if (cache->copier)
2114 dm_kcopyd_client_destroy(cache->copier);
2115
2116 if (cache->cmd)
2117 dm_cache_metadata_close(cache->cmd);
2118
2119 if (cache->metadata_dev)
2120 dm_put_device(cache->ti, cache->metadata_dev);
2121
2122 if (cache->origin_dev)
2123 dm_put_device(cache->ti, cache->origin_dev);
2124
2125 if (cache->cache_dev)
2126 dm_put_device(cache->ti, cache->cache_dev);
2127
2128 if (cache->policy)
2129 dm_cache_policy_destroy(cache->policy);
2130
2131 for (i = 0; i < cache->nr_ctr_args ; i++)
2132 kfree(cache->ctr_args[i]);
2133 kfree(cache->ctr_args);
2134
2135 kfree(cache);
2136}
2137
2138static void cache_dtr(struct dm_target *ti)
2139{
2140 struct cache *cache = ti->private;
2141
2142 destroy(cache);
2143}
2144
2145static sector_t get_dev_size(struct dm_dev *dev)
2146{
2147 return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
2148}
2149
2150/*----------------------------------------------------------------*/
2151
2152/*
2153 * Construct a cache device mapping.
2154 *
2155 * cache <metadata dev> <cache dev> <origin dev> <block size>
2156 * <#feature args> [<feature arg>]*
2157 * <policy> <#policy args> [<policy arg>]*
2158 *
2159 * metadata dev : fast device holding the persistent metadata
2160 * cache dev : fast device holding cached data blocks
2161 * origin dev : slow device holding original data blocks
2162 * block size : cache unit size in sectors
2163 *
2164 * #feature args : number of feature arguments passed
2165 * feature args : writethrough. (The default is writeback.)
2166 *
2167 * policy : the replacement policy to use
2168 * #policy args : an even number of policy arguments corresponding
2169 * to key/value pairs passed to the policy
2170 * policy args : key/value pairs passed to the policy
2171 * E.g. 'sequential_threshold 1024'
2172 * See cache-policies.txt for details.
2173 *
2174 * Optional feature arguments are:
2175 * writethrough : write through caching that prohibits cache block
2176 * content from being different from origin block content.
2177 * Without this argument, the default behaviour is to write
2178 * back cache block contents later for performance reasons,
2179 * so they may differ from the corresponding origin blocks.
2180 */
2181struct cache_args {
2182 struct dm_target *ti;
2183
2184 struct dm_dev *metadata_dev;
2185
2186 struct dm_dev *cache_dev;
2187 sector_t cache_sectors;
2188
2189 struct dm_dev *origin_dev;
2190 sector_t origin_sectors;
2191
2192 uint32_t block_size;
2193
2194 const char *policy_name;
2195 int policy_argc;
2196 const char **policy_argv;
2197
2198 struct cache_features features;
2199};
2200
2201static void destroy_cache_args(struct cache_args *ca)
2202{
2203 if (ca->metadata_dev)
2204 dm_put_device(ca->ti, ca->metadata_dev);
2205
2206 if (ca->cache_dev)
2207 dm_put_device(ca->ti, ca->cache_dev);
2208
2209 if (ca->origin_dev)
2210 dm_put_device(ca->ti, ca->origin_dev);
2211
2212 kfree(ca);
2213}
2214
2215static bool at_least_one_arg(struct dm_arg_set *as, char **error)
2216{
2217 if (!as->argc) {
2218 *error = "Insufficient args";
2219 return false;
2220 }
2221
2222 return true;
2223}
2224
2225static int parse_metadata_dev(struct cache_args *ca, struct dm_arg_set *as,
2226 char **error)
2227{
2228 int r;
2229 sector_t metadata_dev_size;
2230 char b[BDEVNAME_SIZE];
2231
2232 if (!at_least_one_arg(as, error))
2233 return -EINVAL;
2234
2235 r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
2236 &ca->metadata_dev);
2237 if (r) {
2238 *error = "Error opening metadata device";
2239 return r;
2240 }
2241
2242 metadata_dev_size = get_dev_size(ca->metadata_dev);
2243 if (metadata_dev_size > DM_CACHE_METADATA_MAX_SECTORS_WARNING)
2244 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
2245 bdevname(ca->metadata_dev->bdev, b), THIN_METADATA_MAX_SECTORS);
2246
2247 return 0;
2248}
2249
2250static int parse_cache_dev(struct cache_args *ca, struct dm_arg_set *as,
2251 char **error)
2252{
2253 int r;
2254
2255 if (!at_least_one_arg(as, error))
2256 return -EINVAL;
2257
2258 r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
2259 &ca->cache_dev);
2260 if (r) {
2261 *error = "Error opening cache device";
2262 return r;
2263 }
2264 ca->cache_sectors = get_dev_size(ca->cache_dev);
2265
2266 return 0;
2267}
2268
2269static int parse_origin_dev(struct cache_args *ca, struct dm_arg_set *as,
2270 char **error)
2271{
2272 int r;
2273
2274 if (!at_least_one_arg(as, error))
2275 return -EINVAL;
2276
2277 r = dm_get_device(ca->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
2278 &ca->origin_dev);
2279 if (r) {
2280 *error = "Error opening origin device";
2281 return r;
2282 }
2283
2284 ca->origin_sectors = get_dev_size(ca->origin_dev);
2285 if (ca->ti->len > ca->origin_sectors) {
2286 *error = "Device size larger than cached device";
2287 return -EINVAL;
2288 }
2289
2290 return 0;
2291}
2292
2293static int parse_block_size(struct cache_args *ca, struct dm_arg_set *as,
2294 char **error)
2295{
2296 unsigned long block_size;
2297
2298 if (!at_least_one_arg(as, error))
2299 return -EINVAL;
2300
2301 if (kstrtoul(dm_shift_arg(as), 10, &block_size) || !block_size ||
2302 block_size < DATA_DEV_BLOCK_SIZE_MIN_SECTORS ||
2303 block_size > DATA_DEV_BLOCK_SIZE_MAX_SECTORS ||
2304 block_size & (DATA_DEV_BLOCK_SIZE_MIN_SECTORS - 1)) {
2305 *error = "Invalid data block size";
2306 return -EINVAL;
2307 }
2308
2309 if (block_size > ca->cache_sectors) {
2310 *error = "Data block size is larger than the cache device";
2311 return -EINVAL;
2312 }
2313
2314 ca->block_size = block_size;
2315
2316 return 0;
2317}
2318
2319static void init_features(struct cache_features *cf)
2320{
2321 cf->mode = CM_WRITE;
2322 cf->io_mode = CM_IO_WRITEBACK;
2323 cf->metadata_version = 1;
2324}
2325
2326static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
2327 char **error)
2328{
2329 static const struct dm_arg _args[] = {
2330 {0, 2, "Invalid number of cache feature arguments"},
2331 };
2332
2333 int r, mode_ctr = 0;
2334 unsigned argc;
2335 const char *arg;
2336 struct cache_features *cf = &ca->features;
2337
2338 init_features(cf);
2339
2340 r = dm_read_arg_group(_args, as, &argc, error);
2341 if (r)
2342 return -EINVAL;
2343
2344 while (argc--) {
2345 arg = dm_shift_arg(as);
2346
2347 if (!strcasecmp(arg, "writeback")) {
2348 cf->io_mode = CM_IO_WRITEBACK;
2349 mode_ctr++;
2350 }
2351
2352 else if (!strcasecmp(arg, "writethrough")) {
2353 cf->io_mode = CM_IO_WRITETHROUGH;
2354 mode_ctr++;
2355 }
2356
2357 else if (!strcasecmp(arg, "passthrough")) {
2358 cf->io_mode = CM_IO_PASSTHROUGH;
2359 mode_ctr++;
2360 }
2361
2362 else if (!strcasecmp(arg, "metadata2"))
2363 cf->metadata_version = 2;
2364
2365 else {
2366 *error = "Unrecognised cache feature requested";
2367 return -EINVAL;
2368 }
2369 }
2370
2371 if (mode_ctr > 1) {
2372 *error = "Duplicate cache io_mode features requested";
2373 return -EINVAL;
2374 }
2375
2376 return 0;
2377}
2378
2379static int parse_policy(struct cache_args *ca, struct dm_arg_set *as,
2380 char **error)
2381{
2382 static const struct dm_arg _args[] = {
2383 {0, 1024, "Invalid number of policy arguments"},
2384 };
2385
2386 int r;
2387
2388 if (!at_least_one_arg(as, error))
2389 return -EINVAL;
2390
2391 ca->policy_name = dm_shift_arg(as);
2392
2393 r = dm_read_arg_group(_args, as, &ca->policy_argc, error);
2394 if (r)
2395 return -EINVAL;
2396
2397 ca->policy_argv = (const char **)as->argv;
2398 dm_consume_args(as, ca->policy_argc);
2399
2400 return 0;
2401}
2402
2403static int parse_cache_args(struct cache_args *ca, int argc, char **argv,
2404 char **error)
2405{
2406 int r;
2407 struct dm_arg_set as;
2408
2409 as.argc = argc;
2410 as.argv = argv;
2411
2412 r = parse_metadata_dev(ca, &as, error);
2413 if (r)
2414 return r;
2415
2416 r = parse_cache_dev(ca, &as, error);
2417 if (r)
2418 return r;
2419
2420 r = parse_origin_dev(ca, &as, error);
2421 if (r)
2422 return r;
2423
2424 r = parse_block_size(ca, &as, error);
2425 if (r)
2426 return r;
2427
2428 r = parse_features(ca, &as, error);
2429 if (r)
2430 return r;
2431
2432 r = parse_policy(ca, &as, error);
2433 if (r)
2434 return r;
2435
2436 return 0;
2437}
2438
2439/*----------------------------------------------------------------*/
2440
2441static struct kmem_cache *migration_cache;
2442
2443#define NOT_CORE_OPTION 1
2444
2445static int process_config_option(struct cache *cache, const char *key, const char *value)
2446{
2447 unsigned long tmp;
2448
2449 if (!strcasecmp(key, "migration_threshold")) {
2450 if (kstrtoul(value, 10, &tmp))
2451 return -EINVAL;
2452
2453 cache->migration_threshold = tmp;
2454 return 0;
2455 }
2456
2457 return NOT_CORE_OPTION;
2458}
2459
2460static int set_config_value(struct cache *cache, const char *key, const char *value)
2461{
2462 int r = process_config_option(cache, key, value);
2463
2464 if (r == NOT_CORE_OPTION)
2465 r = policy_set_config_value(cache->policy, key, value);
2466
2467 if (r)
2468 DMWARN("bad config value for %s: %s", key, value);
2469
2470 return r;
2471}
2472
2473static int set_config_values(struct cache *cache, int argc, const char **argv)
2474{
2475 int r = 0;
2476
2477 if (argc & 1) {
2478 DMWARN("Odd number of policy arguments given but they should be <key> <value> pairs.");
2479 return -EINVAL;
2480 }
2481
2482 while (argc) {
2483 r = set_config_value(cache, argv[0], argv[1]);
2484 if (r)
2485 break;
2486
2487 argc -= 2;
2488 argv += 2;
2489 }
2490
2491 return r;
2492}
2493
2494static int create_cache_policy(struct cache *cache, struct cache_args *ca,
2495 char **error)
2496{
2497 struct dm_cache_policy *p = dm_cache_policy_create(ca->policy_name,
2498 cache->cache_size,
2499 cache->origin_sectors,
2500 cache->sectors_per_block);
2501 if (IS_ERR(p)) {
2502 *error = "Error creating cache's policy";
2503 return PTR_ERR(p);
2504 }
2505 cache->policy = p;
2506 BUG_ON(!cache->policy);
2507
2508 return 0;
2509}
2510
2511/*
2512 * We want the discard block size to be at least the size of the cache
2513 * block size and have no more than 2^14 discard blocks across the origin.
2514 */
2515#define MAX_DISCARD_BLOCKS (1 << 14)
2516
2517static bool too_many_discard_blocks(sector_t discard_block_size,
2518 sector_t origin_size)
2519{
2520 (void) sector_div(origin_size, discard_block_size);
2521
2522 return origin_size > MAX_DISCARD_BLOCKS;
2523}
2524
2525static sector_t calculate_discard_block_size(sector_t cache_block_size,
2526 sector_t origin_size)
2527{
2528 sector_t discard_block_size = cache_block_size;
2529
2530 if (origin_size)
2531 while (too_many_discard_blocks(discard_block_size, origin_size))
2532 discard_block_size *= 2;
2533
2534 return discard_block_size;
2535}
2536
2537static void set_cache_size(struct cache *cache, dm_cblock_t size)
2538{
2539 dm_block_t nr_blocks = from_cblock(size);
2540
2541 if (nr_blocks > (1 << 20) && cache->cache_size != size)
2542 DMWARN_LIMIT("You have created a cache device with a lot of individual cache blocks (%llu)\n"
2543 "All these mappings can consume a lot of kernel memory, and take some time to read/write.\n"
2544 "Please consider increasing the cache block size to reduce the overall cache block count.",
2545 (unsigned long long) nr_blocks);
2546
2547 cache->cache_size = size;
2548}
2549
2550static int is_congested(struct dm_dev *dev, int bdi_bits)
2551{
2552 struct request_queue *q = bdev_get_queue(dev->bdev);
2553 return bdi_congested(q->backing_dev_info, bdi_bits);
2554}
2555
2556static int cache_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
2557{
2558 struct cache *cache = container_of(cb, struct cache, callbacks);
2559
2560 return is_congested(cache->origin_dev, bdi_bits) ||
2561 is_congested(cache->cache_dev, bdi_bits);
2562}
2563
2564#define DEFAULT_MIGRATION_THRESHOLD 2048
2565
2566static int cache_create(struct cache_args *ca, struct cache **result)
2567{
2568 int r = 0;
2569 char **error = &ca->ti->error;
2570 struct cache *cache;
2571 struct dm_target *ti = ca->ti;
2572 dm_block_t origin_blocks;
2573 struct dm_cache_metadata *cmd;
2574 bool may_format = ca->features.mode == CM_WRITE;
2575
2576 cache = kzalloc(sizeof(*cache), GFP_KERNEL);
2577 if (!cache)
2578 return -ENOMEM;
2579
2580 cache->ti = ca->ti;
2581 ti->private = cache;
2582 ti->num_flush_bios = 2;
2583 ti->flush_supported = true;
2584
2585 ti->num_discard_bios = 1;
2586 ti->discards_supported = true;
2587 ti->split_discard_bios = false;
2588
2589 cache->features = ca->features;
2590 ti->per_io_data_size = get_per_bio_data_size(cache);
2591
2592 cache->callbacks.congested_fn = cache_is_congested;
2593 dm_table_add_target_callbacks(ti->table, &cache->callbacks);
2594
2595 cache->metadata_dev = ca->metadata_dev;
2596 cache->origin_dev = ca->origin_dev;
2597 cache->cache_dev = ca->cache_dev;
2598
2599 ca->metadata_dev = ca->origin_dev = ca->cache_dev = NULL;
2600
2601 origin_blocks = cache->origin_sectors = ca->origin_sectors;
2602 origin_blocks = block_div(origin_blocks, ca->block_size);
2603 cache->origin_blocks = to_oblock(origin_blocks);
2604
2605 cache->sectors_per_block = ca->block_size;
2606 if (dm_set_target_max_io_len(ti, cache->sectors_per_block)) {
2607 r = -EINVAL;
2608 goto bad;
2609 }
2610
2611 if (ca->block_size & (ca->block_size - 1)) {
2612 dm_block_t cache_size = ca->cache_sectors;
2613
2614 cache->sectors_per_block_shift = -1;
2615 cache_size = block_div(cache_size, ca->block_size);
2616 set_cache_size(cache, to_cblock(cache_size));
2617 } else {
2618 cache->sectors_per_block_shift = __ffs(ca->block_size);
2619 set_cache_size(cache, to_cblock(ca->cache_sectors >> cache->sectors_per_block_shift));
2620 }
2621
2622 r = create_cache_policy(cache, ca, error);
2623 if (r)
2624 goto bad;
2625
2626 cache->policy_nr_args = ca->policy_argc;
2627 cache->migration_threshold = DEFAULT_MIGRATION_THRESHOLD;
2628
2629 r = set_config_values(cache, ca->policy_argc, ca->policy_argv);
2630 if (r) {
2631 *error = "Error setting cache policy's config values";
2632 goto bad;
2633 }
2634
2635 cmd = dm_cache_metadata_open(cache->metadata_dev->bdev,
2636 ca->block_size, may_format,
2637 dm_cache_policy_get_hint_size(cache->policy),
2638 ca->features.metadata_version);
2639 if (IS_ERR(cmd)) {
2640 *error = "Error creating metadata object";
2641 r = PTR_ERR(cmd);
2642 goto bad;
2643 }
2644 cache->cmd = cmd;
2645 set_cache_mode(cache, CM_WRITE);
2646 if (get_cache_mode(cache) != CM_WRITE) {
2647 *error = "Unable to get write access to metadata, please check/repair metadata.";
2648 r = -EINVAL;
2649 goto bad;
2650 }
2651
2652 if (passthrough_mode(&cache->features)) {
2653 bool all_clean;
2654
2655 r = dm_cache_metadata_all_clean(cache->cmd, &all_clean);
2656 if (r) {
2657 *error = "dm_cache_metadata_all_clean() failed";
2658 goto bad;
2659 }
2660
2661 if (!all_clean) {
2662 *error = "Cannot enter passthrough mode unless all blocks are clean";
2663 r = -EINVAL;
2664 goto bad;
2665 }
2666
2667 policy_allow_migrations(cache->policy, false);
2668 }
2669
2670 spin_lock_init(&cache->lock);
2671 INIT_LIST_HEAD(&cache->deferred_cells);
2672 bio_list_init(&cache->deferred_bios);
2673 bio_list_init(&cache->deferred_writethrough_bios);
2674 atomic_set(&cache->nr_allocated_migrations, 0);
2675 atomic_set(&cache->nr_io_migrations, 0);
2676 init_waitqueue_head(&cache->migration_wait);
2677
2678 r = -ENOMEM;
2679 atomic_set(&cache->nr_dirty, 0);
2680 cache->dirty_bitset = alloc_bitset(from_cblock(cache->cache_size));
2681 if (!cache->dirty_bitset) {
2682 *error = "could not allocate dirty bitset";
2683 goto bad;
2684 }
2685 clear_bitset(cache->dirty_bitset, from_cblock(cache->cache_size));
2686
2687 cache->discard_block_size =
2688 calculate_discard_block_size(cache->sectors_per_block,
2689 cache->origin_sectors);
2690 cache->discard_nr_blocks = to_dblock(dm_sector_div_up(cache->origin_sectors,
2691 cache->discard_block_size));
2692 cache->discard_bitset = alloc_bitset(from_dblock(cache->discard_nr_blocks));
2693 if (!cache->discard_bitset) {
2694 *error = "could not allocate discard bitset";
2695 goto bad;
2696 }
2697 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
2698
2699 cache->copier = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2700 if (IS_ERR(cache->copier)) {
2701 *error = "could not create kcopyd client";
2702 r = PTR_ERR(cache->copier);
2703 goto bad;
2704 }
2705
2706 cache->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
2707 if (!cache->wq) {
2708 *error = "could not create workqueue for metadata object";
2709 goto bad;
2710 }
2711 INIT_WORK(&cache->deferred_bio_worker, process_deferred_bios);
2712 INIT_WORK(&cache->deferred_writethrough_worker,
2713 process_deferred_writethrough_bios);
2714 INIT_WORK(&cache->migration_worker, check_migrations);
2715 INIT_DELAYED_WORK(&cache->waker, do_waker);
2716
2717 cache->prison = dm_bio_prison_create_v2(cache->wq);
2718 if (!cache->prison) {
2719 *error = "could not create bio prison";
2720 goto bad;
2721 }
2722
2723 cache->migration_pool = mempool_create_slab_pool(MIGRATION_POOL_SIZE,
2724 migration_cache);
2725 if (!cache->migration_pool) {
2726 *error = "Error creating cache's migration mempool";
2727 goto bad;
2728 }
2729
2730 cache->need_tick_bio = true;
2731 cache->sized = false;
2732 cache->invalidate = false;
2733 cache->commit_requested = false;
2734 cache->loaded_mappings = false;
2735 cache->loaded_discards = false;
2736
2737 load_stats(cache);
2738
2739 atomic_set(&cache->stats.demotion, 0);
2740 atomic_set(&cache->stats.promotion, 0);
2741 atomic_set(&cache->stats.copies_avoided, 0);
2742 atomic_set(&cache->stats.cache_cell_clash, 0);
2743 atomic_set(&cache->stats.commit_count, 0);
2744 atomic_set(&cache->stats.discard_count, 0);
2745
2746 spin_lock_init(&cache->invalidation_lock);
2747 INIT_LIST_HEAD(&cache->invalidation_requests);
2748
2749 batcher_init(&cache->committer, commit_op, cache,
2750 issue_op, cache, cache->wq);
2751 iot_init(&cache->tracker);
2752
2753 init_rwsem(&cache->background_work_lock);
2754 prevent_background_work(cache);
2755
2756 *result = cache;
2757 return 0;
2758bad:
2759 destroy(cache);
2760 return r;
2761}
2762
2763static int copy_ctr_args(struct cache *cache, int argc, const char **argv)
2764{
2765 unsigned i;
2766 const char **copy;
2767
2768 copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
2769 if (!copy)
2770 return -ENOMEM;
2771 for (i = 0; i < argc; i++) {
2772 copy[i] = kstrdup(argv[i], GFP_KERNEL);
2773 if (!copy[i]) {
2774 while (i--)
2775 kfree(copy[i]);
2776 kfree(copy);
2777 return -ENOMEM;
2778 }
2779 }
2780
2781 cache->nr_ctr_args = argc;
2782 cache->ctr_args = copy;
2783
2784 return 0;
2785}
2786
2787static int cache_ctr(struct dm_target *ti, unsigned argc, char **argv)
2788{
2789 int r = -EINVAL;
2790 struct cache_args *ca;
2791 struct cache *cache = NULL;
2792
2793 ca = kzalloc(sizeof(*ca), GFP_KERNEL);
2794 if (!ca) {
2795 ti->error = "Error allocating memory for cache";
2796 return -ENOMEM;
2797 }
2798 ca->ti = ti;
2799
2800 r = parse_cache_args(ca, argc, argv, &ti->error);
2801 if (r)
2802 goto out;
2803
2804 r = cache_create(ca, &cache);
2805 if (r)
2806 goto out;
2807
2808 r = copy_ctr_args(cache, argc - 3, (const char **)argv + 3);
2809 if (r) {
2810 destroy(cache);
2811 goto out;
2812 }
2813
2814 ti->private = cache;
2815out:
2816 destroy_cache_args(ca);
2817 return r;
2818}
2819
2820/*----------------------------------------------------------------*/
2821
2822static int cache_map(struct dm_target *ti, struct bio *bio)
2823{
2824 struct cache *cache = ti->private;
2825
2826 int r;
2827 bool commit_needed;
2828 dm_oblock_t block = get_bio_block(cache, bio);
2829 size_t pb_data_size = get_per_bio_data_size(cache);
2830
2831 init_per_bio_data(bio, pb_data_size);
2832 if (unlikely(from_oblock(block) >= from_oblock(cache->origin_blocks))) {
2833 /*
2834 * This can only occur if the io goes to a partial block at
2835 * the end of the origin device. We don't cache these.
2836 * Just remap to the origin and carry on.
2837 */
2838 remap_to_origin(cache, bio);
2839 accounted_begin(cache, bio);
2840 return DM_MAPIO_REMAPPED;
2841 }
2842
2843 if (discard_or_flush(bio)) {
2844 defer_bio(cache, bio);
2845 return DM_MAPIO_SUBMITTED;
2846 }
2847
2848 r = map_bio(cache, bio, block, &commit_needed);
2849 if (commit_needed)
2850 schedule_commit(&cache->committer);
2851
2852 return r;
2853}
2854
2855static int cache_end_io(struct dm_target *ti, struct bio *bio,
2856 blk_status_t *error)
2857{
2858 struct cache *cache = ti->private;
2859 unsigned long flags;
2860 size_t pb_data_size = get_per_bio_data_size(cache);
2861 struct per_bio_data *pb = get_per_bio_data(bio, pb_data_size);
2862
2863 if (pb->tick) {
2864 policy_tick(cache->policy, false);
2865
2866 spin_lock_irqsave(&cache->lock, flags);
2867 cache->need_tick_bio = true;
2868 spin_unlock_irqrestore(&cache->lock, flags);
2869 }
2870
2871 bio_drop_shared_lock(cache, bio);
2872 accounted_complete(cache, bio);
2873
2874 return DM_ENDIO_DONE;
2875}
2876
2877static int write_dirty_bitset(struct cache *cache)
2878{
2879 int r;
2880
2881 if (get_cache_mode(cache) >= CM_READ_ONLY)
2882 return -EINVAL;
2883
2884 r = dm_cache_set_dirty_bits(cache->cmd, from_cblock(cache->cache_size), cache->dirty_bitset);
2885 if (r)
2886 metadata_operation_failed(cache, "dm_cache_set_dirty_bits", r);
2887
2888 return r;
2889}
2890
2891static int write_discard_bitset(struct cache *cache)
2892{
2893 unsigned i, r;
2894
2895 if (get_cache_mode(cache) >= CM_READ_ONLY)
2896 return -EINVAL;
2897
2898 r = dm_cache_discard_bitset_resize(cache->cmd, cache->discard_block_size,
2899 cache->discard_nr_blocks);
2900 if (r) {
2901 DMERR("%s: could not resize on-disk discard bitset", cache_device_name(cache));
2902 metadata_operation_failed(cache, "dm_cache_discard_bitset_resize", r);
2903 return r;
2904 }
2905
2906 for (i = 0; i < from_dblock(cache->discard_nr_blocks); i++) {
2907 r = dm_cache_set_discard(cache->cmd, to_dblock(i),
2908 is_discarded(cache, to_dblock(i)));
2909 if (r) {
2910 metadata_operation_failed(cache, "dm_cache_set_discard", r);
2911 return r;
2912 }
2913 }
2914
2915 return 0;
2916}
2917
2918static int write_hints(struct cache *cache)
2919{
2920 int r;
2921
2922 if (get_cache_mode(cache) >= CM_READ_ONLY)
2923 return -EINVAL;
2924
2925 r = dm_cache_write_hints(cache->cmd, cache->policy);
2926 if (r) {
2927 metadata_operation_failed(cache, "dm_cache_write_hints", r);
2928 return r;
2929 }
2930
2931 return 0;
2932}
2933
2934/*
2935 * returns true on success
2936 */
2937static bool sync_metadata(struct cache *cache)
2938{
2939 int r1, r2, r3, r4;
2940
2941 r1 = write_dirty_bitset(cache);
2942 if (r1)
2943 DMERR("%s: could not write dirty bitset", cache_device_name(cache));
2944
2945 r2 = write_discard_bitset(cache);
2946 if (r2)
2947 DMERR("%s: could not write discard bitset", cache_device_name(cache));
2948
2949 save_stats(cache);
2950
2951 r3 = write_hints(cache);
2952 if (r3)
2953 DMERR("%s: could not write hints", cache_device_name(cache));
2954
2955 /*
2956 * If writing the above metadata failed, we still commit, but don't
2957 * set the clean shutdown flag. This will effectively force every
2958 * dirty bit to be set on reload.
2959 */
2960 r4 = commit(cache, !r1 && !r2 && !r3);
2961 if (r4)
2962 DMERR("%s: could not write cache metadata", cache_device_name(cache));
2963
2964 return !r1 && !r2 && !r3 && !r4;
2965}
2966
2967static void cache_postsuspend(struct dm_target *ti)
2968{
2969 struct cache *cache = ti->private;
2970
2971 prevent_background_work(cache);
2972 BUG_ON(atomic_read(&cache->nr_io_migrations));
2973
2974 cancel_delayed_work(&cache->waker);
2975 flush_workqueue(cache->wq);
2976 WARN_ON(cache->tracker.in_flight);
2977
2978 /*
2979 * If it's a flush suspend there won't be any deferred bios, so this
2980 * call is harmless.
2981 */
2982 requeue_deferred_bios(cache);
2983
2984 if (get_cache_mode(cache) == CM_WRITE)
2985 (void) sync_metadata(cache);
2986}
2987
2988static int load_mapping(void *context, dm_oblock_t oblock, dm_cblock_t cblock,
2989 bool dirty, uint32_t hint, bool hint_valid)
2990{
2991 int r;
2992 struct cache *cache = context;
2993
2994 if (dirty) {
2995 set_bit(from_cblock(cblock), cache->dirty_bitset);
2996 atomic_inc(&cache->nr_dirty);
2997 } else
2998 clear_bit(from_cblock(cblock), cache->dirty_bitset);
2999
3000 r = policy_load_mapping(cache->policy, oblock, cblock, dirty, hint, hint_valid);
3001 if (r)
3002 return r;
3003
3004 return 0;
3005}
3006
3007/*
3008 * The discard block size in the on disk metadata is not
3009 * neccessarily the same as we're currently using. So we have to
3010 * be careful to only set the discarded attribute if we know it
3011 * covers a complete block of the new size.
3012 */
3013struct discard_load_info {
3014 struct cache *cache;
3015
3016 /*
3017 * These blocks are sized using the on disk dblock size, rather
3018 * than the current one.
3019 */
3020 dm_block_t block_size;
3021 dm_block_t discard_begin, discard_end;
3022};
3023
3024static void discard_load_info_init(struct cache *cache,
3025 struct discard_load_info *li)
3026{
3027 li->cache = cache;
3028 li->discard_begin = li->discard_end = 0;
3029}
3030
3031static void set_discard_range(struct discard_load_info *li)
3032{
3033 sector_t b, e;
3034
3035 if (li->discard_begin == li->discard_end)
3036 return;
3037
3038 /*
3039 * Convert to sectors.
3040 */
3041 b = li->discard_begin * li->block_size;
3042 e = li->discard_end * li->block_size;
3043
3044 /*
3045 * Then convert back to the current dblock size.
3046 */
3047 b = dm_sector_div_up(b, li->cache->discard_block_size);
3048 sector_div(e, li->cache->discard_block_size);
3049
3050 /*
3051 * The origin may have shrunk, so we need to check we're still in
3052 * bounds.
3053 */
3054 if (e > from_dblock(li->cache->discard_nr_blocks))
3055 e = from_dblock(li->cache->discard_nr_blocks);
3056
3057 for (; b < e; b++)
3058 set_discard(li->cache, to_dblock(b));
3059}
3060
3061static int load_discard(void *context, sector_t discard_block_size,
3062 dm_dblock_t dblock, bool discard)
3063{
3064 struct discard_load_info *li = context;
3065
3066 li->block_size = discard_block_size;
3067
3068 if (discard) {
3069 if (from_dblock(dblock) == li->discard_end)
3070 /*
3071 * We're already in a discard range, just extend it.
3072 */
3073 li->discard_end = li->discard_end + 1ULL;
3074
3075 else {
3076 /*
3077 * Emit the old range and start a new one.
3078 */
3079 set_discard_range(li);
3080 li->discard_begin = from_dblock(dblock);
3081 li->discard_end = li->discard_begin + 1ULL;
3082 }
3083 } else {
3084 set_discard_range(li);
3085 li->discard_begin = li->discard_end = 0;
3086 }
3087
3088 return 0;
3089}
3090
3091static dm_cblock_t get_cache_dev_size(struct cache *cache)
3092{
3093 sector_t size = get_dev_size(cache->cache_dev);
3094 (void) sector_div(size, cache->sectors_per_block);
3095 return to_cblock(size);
3096}
3097
3098static bool can_resize(struct cache *cache, dm_cblock_t new_size)
3099{
3100 if (from_cblock(new_size) > from_cblock(cache->cache_size)) {
3101 if (cache->sized) {
3102 DMERR("%s: unable to extend cache due to missing cache table reload",
3103 cache_device_name(cache));
3104 return false;
3105 }
3106 }
3107
3108 /*
3109 * We can't drop a dirty block when shrinking the cache.
3110 */
3111 while (from_cblock(new_size) < from_cblock(cache->cache_size)) {
3112 new_size = to_cblock(from_cblock(new_size) + 1);
3113 if (is_dirty(cache, new_size)) {
3114 DMERR("%s: unable to shrink cache; cache block %llu is dirty",
3115 cache_device_name(cache),
3116 (unsigned long long) from_cblock(new_size));
3117 return false;
3118 }
3119 }
3120
3121 return true;
3122}
3123
3124static int resize_cache_dev(struct cache *cache, dm_cblock_t new_size)
3125{
3126 int r;
3127
3128 r = dm_cache_resize(cache->cmd, new_size);
3129 if (r) {
3130 DMERR("%s: could not resize cache metadata", cache_device_name(cache));
3131 metadata_operation_failed(cache, "dm_cache_resize", r);
3132 return r;
3133 }
3134
3135 set_cache_size(cache, new_size);
3136
3137 return 0;
3138}
3139
3140static int cache_preresume(struct dm_target *ti)
3141{
3142 int r = 0;
3143 struct cache *cache = ti->private;
3144 dm_cblock_t csize = get_cache_dev_size(cache);
3145
3146 /*
3147 * Check to see if the cache has resized.
3148 */
3149 if (!cache->sized) {
3150 r = resize_cache_dev(cache, csize);
3151 if (r)
3152 return r;
3153
3154 cache->sized = true;
3155
3156 } else if (csize != cache->cache_size) {
3157 if (!can_resize(cache, csize))
3158 return -EINVAL;
3159
3160 r = resize_cache_dev(cache, csize);
3161 if (r)
3162 return r;
3163 }
3164
3165 if (!cache->loaded_mappings) {
3166 r = dm_cache_load_mappings(cache->cmd, cache->policy,
3167 load_mapping, cache);
3168 if (r) {
3169 DMERR("%s: could not load cache mappings", cache_device_name(cache));
3170 metadata_operation_failed(cache, "dm_cache_load_mappings", r);
3171 return r;
3172 }
3173
3174 cache->loaded_mappings = true;
3175 }
3176
3177 if (!cache->loaded_discards) {
3178 struct discard_load_info li;
3179
3180 /*
3181 * The discard bitset could have been resized, or the
3182 * discard block size changed. To be safe we start by
3183 * setting every dblock to not discarded.
3184 */
3185 clear_bitset(cache->discard_bitset, from_dblock(cache->discard_nr_blocks));
3186
3187 discard_load_info_init(cache, &li);
3188 r = dm_cache_load_discards(cache->cmd, load_discard, &li);
3189 if (r) {
3190 DMERR("%s: could not load origin discards", cache_device_name(cache));
3191 metadata_operation_failed(cache, "dm_cache_load_discards", r);
3192 return r;
3193 }
3194 set_discard_range(&li);
3195
3196 cache->loaded_discards = true;
3197 }
3198
3199 return r;
3200}
3201
3202static void cache_resume(struct dm_target *ti)
3203{
3204 struct cache *cache = ti->private;
3205
3206 cache->need_tick_bio = true;
3207 allow_background_work(cache);
3208 do_waker(&cache->waker.work);
3209}
3210
3211/*
3212 * Status format:
3213 *
3214 * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
3215 * <cache block size> <#used cache blocks>/<#total cache blocks>
3216 * <#read hits> <#read misses> <#write hits> <#write misses>
3217 * <#demotions> <#promotions> <#dirty>
3218 * <#features> <features>*
3219 * <#core args> <core args>
3220 * <policy name> <#policy args> <policy args>* <cache metadata mode> <needs_check>
3221 */
3222static void cache_status(struct dm_target *ti, status_type_t type,
3223 unsigned status_flags, char *result, unsigned maxlen)
3224{
3225 int r = 0;
3226 unsigned i;
3227 ssize_t sz = 0;
3228 dm_block_t nr_free_blocks_metadata = 0;
3229 dm_block_t nr_blocks_metadata = 0;
3230 char buf[BDEVNAME_SIZE];
3231 struct cache *cache = ti->private;
3232 dm_cblock_t residency;
3233 bool needs_check;
3234
3235 switch (type) {
3236 case STATUSTYPE_INFO:
3237 if (get_cache_mode(cache) == CM_FAIL) {
3238 DMEMIT("Fail");
3239 break;
3240 }
3241
3242 /* Commit to ensure statistics aren't out-of-date */
3243 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
3244 (void) commit(cache, false);
3245
3246 r = dm_cache_get_free_metadata_block_count(cache->cmd, &nr_free_blocks_metadata);
3247 if (r) {
3248 DMERR("%s: dm_cache_get_free_metadata_block_count returned %d",
3249 cache_device_name(cache), r);
3250 goto err;
3251 }
3252
3253 r = dm_cache_get_metadata_dev_size(cache->cmd, &nr_blocks_metadata);
3254 if (r) {
3255 DMERR("%s: dm_cache_get_metadata_dev_size returned %d",
3256 cache_device_name(cache), r);
3257 goto err;
3258 }
3259
3260 residency = policy_residency(cache->policy);
3261
3262 DMEMIT("%u %llu/%llu %llu %llu/%llu %u %u %u %u %u %u %lu ",
3263 (unsigned)DM_CACHE_METADATA_BLOCK_SIZE,
3264 (unsigned long long)(nr_blocks_metadata - nr_free_blocks_metadata),
3265 (unsigned long long)nr_blocks_metadata,
3266 (unsigned long long)cache->sectors_per_block,
3267 (unsigned long long) from_cblock(residency),
3268 (unsigned long long) from_cblock(cache->cache_size),
3269 (unsigned) atomic_read(&cache->stats.read_hit),
3270 (unsigned) atomic_read(&cache->stats.read_miss),
3271 (unsigned) atomic_read(&cache->stats.write_hit),
3272 (unsigned) atomic_read(&cache->stats.write_miss),
3273 (unsigned) atomic_read(&cache->stats.demotion),
3274 (unsigned) atomic_read(&cache->stats.promotion),
3275 (unsigned long) atomic_read(&cache->nr_dirty));
3276
3277 if (cache->features.metadata_version == 2)
3278 DMEMIT("2 metadata2 ");
3279 else
3280 DMEMIT("1 ");
3281
3282 if (writethrough_mode(&cache->features))
3283 DMEMIT("writethrough ");
3284
3285 else if (passthrough_mode(&cache->features))
3286 DMEMIT("passthrough ");
3287
3288 else if (writeback_mode(&cache->features))
3289 DMEMIT("writeback ");
3290
3291 else {
3292 DMERR("%s: internal error: unknown io mode: %d",
3293 cache_device_name(cache), (int) cache->features.io_mode);
3294 goto err;
3295 }
3296
3297 DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold);
3298
3299 DMEMIT("%s ", dm_cache_policy_get_name(cache->policy));
3300 if (sz < maxlen) {
3301 r = policy_emit_config_values(cache->policy, result, maxlen, &sz);
3302 if (r)
3303 DMERR("%s: policy_emit_config_values returned %d",
3304 cache_device_name(cache), r);
3305 }
3306
3307 if (get_cache_mode(cache) == CM_READ_ONLY)
3308 DMEMIT("ro ");
3309 else
3310 DMEMIT("rw ");
3311
3312 r = dm_cache_metadata_needs_check(cache->cmd, &needs_check);
3313
3314 if (r || needs_check)
3315 DMEMIT("needs_check ");
3316 else
3317 DMEMIT("- ");
3318
3319 break;
3320
3321 case STATUSTYPE_TABLE:
3322 format_dev_t(buf, cache->metadata_dev->bdev->bd_dev);
3323 DMEMIT("%s ", buf);
3324 format_dev_t(buf, cache->cache_dev->bdev->bd_dev);
3325 DMEMIT("%s ", buf);
3326 format_dev_t(buf, cache->origin_dev->bdev->bd_dev);
3327 DMEMIT("%s", buf);
3328
3329 for (i = 0; i < cache->nr_ctr_args - 1; i++)
3330 DMEMIT(" %s", cache->ctr_args[i]);
3331 if (cache->nr_ctr_args)
3332 DMEMIT(" %s", cache->ctr_args[cache->nr_ctr_args - 1]);
3333 }
3334
3335 return;
3336
3337err:
3338 DMEMIT("Error");
3339}
3340
3341/*
3342 * Defines a range of cblocks, begin to (end - 1) are in the range. end is
3343 * the one-past-the-end value.
3344 */
3345struct cblock_range {
3346 dm_cblock_t begin;
3347 dm_cblock_t end;
3348};
3349
3350/*
3351 * A cache block range can take two forms:
3352 *
3353 * i) A single cblock, eg. '3456'
3354 * ii) A begin and end cblock with a dash between, eg. 123-234
3355 */
3356static int parse_cblock_range(struct cache *cache, const char *str,
3357 struct cblock_range *result)
3358{
3359 char dummy;
3360 uint64_t b, e;
3361 int r;
3362
3363 /*
3364 * Try and parse form (ii) first.
3365 */
3366 r = sscanf(str, "%llu-%llu%c", &b, &e, &dummy);
3367 if (r < 0)
3368 return r;
3369
3370 if (r == 2) {
3371 result->begin = to_cblock(b);
3372 result->end = to_cblock(e);
3373 return 0;
3374 }
3375
3376 /*
3377 * That didn't work, try form (i).
3378 */
3379 r = sscanf(str, "%llu%c", &b, &dummy);
3380 if (r < 0)
3381 return r;
3382
3383 if (r == 1) {
3384 result->begin = to_cblock(b);
3385 result->end = to_cblock(from_cblock(result->begin) + 1u);
3386 return 0;
3387 }
3388
3389 DMERR("%s: invalid cblock range '%s'", cache_device_name(cache), str);
3390 return -EINVAL;
3391}
3392
3393static int validate_cblock_range(struct cache *cache, struct cblock_range *range)
3394{
3395 uint64_t b = from_cblock(range->begin);
3396 uint64_t e = from_cblock(range->end);
3397 uint64_t n = from_cblock(cache->cache_size);
3398
3399 if (b >= n) {
3400 DMERR("%s: begin cblock out of range: %llu >= %llu",
3401 cache_device_name(cache), b, n);
3402 return -EINVAL;
3403 }
3404
3405 if (e > n) {
3406 DMERR("%s: end cblock out of range: %llu > %llu",
3407 cache_device_name(cache), e, n);
3408 return -EINVAL;
3409 }
3410
3411 if (b >= e) {
3412 DMERR("%s: invalid cblock range: %llu >= %llu",
3413 cache_device_name(cache), b, e);
3414 return -EINVAL;
3415 }
3416
3417 return 0;
3418}
3419
3420static inline dm_cblock_t cblock_succ(dm_cblock_t b)
3421{
3422 return to_cblock(from_cblock(b) + 1);
3423}
3424
3425static int request_invalidation(struct cache *cache, struct cblock_range *range)
3426{
3427 int r = 0;
3428
3429 /*
3430 * We don't need to do any locking here because we know we're in
3431 * passthrough mode. There's is potential for a race between an
3432 * invalidation triggered by an io and an invalidation message. This
3433 * is harmless, we must not worry if the policy call fails.
3434 */
3435 while (range->begin != range->end) {
3436 r = invalidate_cblock(cache, range->begin);
3437 if (r)
3438 return r;
3439
3440 range->begin = cblock_succ(range->begin);
3441 }
3442
3443 cache->commit_requested = true;
3444 return r;
3445}
3446
3447static int process_invalidate_cblocks_message(struct cache *cache, unsigned count,
3448 const char **cblock_ranges)
3449{
3450 int r = 0;
3451 unsigned i;
3452 struct cblock_range range;
3453
3454 if (!passthrough_mode(&cache->features)) {
3455 DMERR("%s: cache has to be in passthrough mode for invalidation",
3456 cache_device_name(cache));
3457 return -EPERM;
3458 }
3459
3460 for (i = 0; i < count; i++) {
3461 r = parse_cblock_range(cache, cblock_ranges[i], &range);
3462 if (r)
3463 break;
3464
3465 r = validate_cblock_range(cache, &range);
3466 if (r)
3467 break;
3468
3469 /*
3470 * Pass begin and end origin blocks to the worker and wake it.
3471 */
3472 r = request_invalidation(cache, &range);
3473 if (r)
3474 break;
3475 }
3476
3477 return r;
3478}
3479
3480/*
3481 * Supports
3482 * "<key> <value>"
3483 * and
3484 * "invalidate_cblocks [(<begin>)|(<begin>-<end>)]*
3485 *
3486 * The key migration_threshold is supported by the cache target core.
3487 */
3488static int cache_message(struct dm_target *ti, unsigned argc, char **argv)
3489{
3490 struct cache *cache = ti->private;
3491
3492 if (!argc)
3493 return -EINVAL;
3494
3495 if (get_cache_mode(cache) >= CM_READ_ONLY) {
3496 DMERR("%s: unable to service cache target messages in READ_ONLY or FAIL mode",
3497 cache_device_name(cache));
3498 return -EOPNOTSUPP;
3499 }
3500
3501 if (!strcasecmp(argv[0], "invalidate_cblocks"))
3502 return process_invalidate_cblocks_message(cache, argc - 1, (const char **) argv + 1);
3503
3504 if (argc != 2)
3505 return -EINVAL;
3506
3507 return set_config_value(cache, argv[0], argv[1]);
3508}
3509
3510static int cache_iterate_devices(struct dm_target *ti,
3511 iterate_devices_callout_fn fn, void *data)
3512{
3513 int r = 0;
3514 struct cache *cache = ti->private;
3515
3516 r = fn(ti, cache->cache_dev, 0, get_dev_size(cache->cache_dev), data);
3517 if (!r)
3518 r = fn(ti, cache->origin_dev, 0, ti->len, data);
3519
3520 return r;
3521}
3522
3523static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
3524{
3525 /*
3526 * FIXME: these limits may be incompatible with the cache device
3527 */
3528 limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024,
3529 cache->origin_sectors);
3530 limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
3531}
3532
3533static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
3534{
3535 struct cache *cache = ti->private;
3536 uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
3537
3538 /*
3539 * If the system-determined stacked limits are compatible with the
3540 * cache's blocksize (io_opt is a factor) do not override them.
3541 */
3542 if (io_opt_sectors < cache->sectors_per_block ||
3543 do_div(io_opt_sectors, cache->sectors_per_block)) {
3544 blk_limits_io_min(limits, cache->sectors_per_block << SECTOR_SHIFT);
3545 blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT);
3546 }
3547 set_discard_limits(cache, limits);
3548}
3549
3550/*----------------------------------------------------------------*/
3551
3552static struct target_type cache_target = {
3553 .name = "cache",
3554 .version = {2, 0, 0},
3555 .module = THIS_MODULE,
3556 .ctr = cache_ctr,
3557 .dtr = cache_dtr,
3558 .map = cache_map,
3559 .end_io = cache_end_io,
3560 .postsuspend = cache_postsuspend,
3561 .preresume = cache_preresume,
3562 .resume = cache_resume,
3563 .status = cache_status,
3564 .message = cache_message,
3565 .iterate_devices = cache_iterate_devices,
3566 .io_hints = cache_io_hints,
3567};
3568
3569static int __init dm_cache_init(void)
3570{
3571 int r;
3572
3573 migration_cache = KMEM_CACHE(dm_cache_migration, 0);
3574 if (!migration_cache)
3575 return -ENOMEM;
3576
3577 r = dm_register_target(&cache_target);
3578 if (r) {
3579 DMERR("cache target registration failed: %d", r);
3580 kmem_cache_destroy(migration_cache);
3581 return r;
3582 }
3583
3584 return 0;
3585}
3586
3587static void __exit dm_cache_exit(void)
3588{
3589 dm_unregister_target(&cache_target);
3590 kmem_cache_destroy(migration_cache);
3591}
3592
3593module_init(dm_cache_init);
3594module_exit(dm_cache_exit);
3595
3596MODULE_DESCRIPTION(DM_NAME " cache target");
3597MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
3598MODULE_LICENSE("GPL");