GNU Linux-libre 4.19.264-gnu1
[releases.git] / drivers / md / dm-bufio.c
1 /*
2  * Copyright (C) 2009-2011 Red Hat, Inc.
3  *
4  * Author: Mikulas Patocka <mpatocka@redhat.com>
5  *
6  * This file is released under the GPL.
7  */
8
9 #include <linux/dm-bufio.h>
10
11 #include <linux/device-mapper.h>
12 #include <linux/dm-io.h>
13 #include <linux/slab.h>
14 #include <linux/sched/mm.h>
15 #include <linux/jiffies.h>
16 #include <linux/vmalloc.h>
17 #include <linux/shrinker.h>
18 #include <linux/module.h>
19 #include <linux/rbtree.h>
20 #include <linux/stacktrace.h>
21
22 #define DM_MSG_PREFIX "bufio"
23
24 /*
25  * Memory management policy:
26  *      Limit the number of buffers to DM_BUFIO_MEMORY_PERCENT of main memory
27  *      or DM_BUFIO_VMALLOC_PERCENT of vmalloc memory (whichever is lower).
28  *      Always allocate at least DM_BUFIO_MIN_BUFFERS buffers.
29  *      Start background writeback when there are DM_BUFIO_WRITEBACK_PERCENT
30  *      dirty buffers.
31  */
32 #define DM_BUFIO_MIN_BUFFERS            8
33
34 #define DM_BUFIO_MEMORY_PERCENT         2
35 #define DM_BUFIO_VMALLOC_PERCENT        25
36 #define DM_BUFIO_WRITEBACK_PERCENT      75
37
38 /*
39  * Check buffer ages in this interval (seconds)
40  */
41 #define DM_BUFIO_WORK_TIMER_SECS        30
42
43 /*
44  * Free buffers when they are older than this (seconds)
45  */
46 #define DM_BUFIO_DEFAULT_AGE_SECS       300
47
48 /*
49  * The nr of bytes of cached data to keep around.
50  */
51 #define DM_BUFIO_DEFAULT_RETAIN_BYTES   (256 * 1024)
52
53 /*
54  * Align buffer writes to this boundary.
55  * Tests show that SSDs have the highest IOPS when using 4k writes.
56  */
57 #define DM_BUFIO_WRITE_ALIGN            4096
58
59 /*
60  * dm_buffer->list_mode
61  */
62 #define LIST_CLEAN      0
63 #define LIST_DIRTY      1
64 #define LIST_SIZE       2
65
66 /*
67  * Linking of buffers:
68  *      All buffers are linked to cache_hash with their hash_list field.
69  *
70  *      Clean buffers that are not being written (B_WRITING not set)
71  *      are linked to lru[LIST_CLEAN] with their lru_list field.
72  *
73  *      Dirty and clean buffers that are being written are linked to
74  *      lru[LIST_DIRTY] with their lru_list field. When the write
75  *      finishes, the buffer cannot be relinked immediately (because we
76  *      are in an interrupt context and relinking requires process
77  *      context), so some clean-not-writing buffers can be held on
78  *      dirty_lru too.  They are later added to lru in the process
79  *      context.
80  */
81 struct dm_bufio_client {
82         struct mutex lock;
83
84         struct list_head lru[LIST_SIZE];
85         unsigned long n_buffers[LIST_SIZE];
86
87         struct block_device *bdev;
88         unsigned block_size;
89         s8 sectors_per_block_bits;
90         void (*alloc_callback)(struct dm_buffer *);
91         void (*write_callback)(struct dm_buffer *);
92
93         struct kmem_cache *slab_buffer;
94         struct kmem_cache *slab_cache;
95         struct dm_io_client *dm_io;
96
97         struct list_head reserved_buffers;
98         unsigned need_reserved_buffers;
99
100         unsigned minimum_buffers;
101
102         struct rb_root buffer_tree;
103         wait_queue_head_t free_buffer_wait;
104
105         sector_t start;
106
107         int async_write_error;
108
109         struct list_head client_list;
110         struct shrinker shrinker;
111 };
112
113 /*
114  * Buffer state bits.
115  */
116 #define B_READING       0
117 #define B_WRITING       1
118 #define B_DIRTY         2
119
120 /*
121  * Describes how the block was allocated:
122  * kmem_cache_alloc(), __get_free_pages() or vmalloc().
123  * See the comment at alloc_buffer_data.
124  */
125 enum data_mode {
126         DATA_MODE_SLAB = 0,
127         DATA_MODE_GET_FREE_PAGES = 1,
128         DATA_MODE_VMALLOC = 2,
129         DATA_MODE_LIMIT = 3
130 };
131
132 struct dm_buffer {
133         struct rb_node node;
134         struct list_head lru_list;
135         sector_t block;
136         void *data;
137         unsigned char data_mode;                /* DATA_MODE_* */
138         unsigned char list_mode;                /* LIST_* */
139         blk_status_t read_error;
140         blk_status_t write_error;
141         unsigned hold_count;
142         unsigned long state;
143         unsigned long last_accessed;
144         unsigned dirty_start;
145         unsigned dirty_end;
146         unsigned write_start;
147         unsigned write_end;
148         struct dm_bufio_client *c;
149         struct list_head write_list;
150         void (*end_io)(struct dm_buffer *, blk_status_t);
151 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
152 #define MAX_STACK 10
153         struct stack_trace stack_trace;
154         unsigned long stack_entries[MAX_STACK];
155 #endif
156 };
157
158 /*----------------------------------------------------------------*/
159
160 #define dm_bufio_in_request()   (!!current->bio_list)
161
162 static void dm_bufio_lock(struct dm_bufio_client *c)
163 {
164         mutex_lock_nested(&c->lock, dm_bufio_in_request());
165 }
166
167 static int dm_bufio_trylock(struct dm_bufio_client *c)
168 {
169         return mutex_trylock(&c->lock);
170 }
171
172 static void dm_bufio_unlock(struct dm_bufio_client *c)
173 {
174         mutex_unlock(&c->lock);
175 }
176
177 /*----------------------------------------------------------------*/
178
179 /*
180  * Default cache size: available memory divided by the ratio.
181  */
182 static unsigned long dm_bufio_default_cache_size;
183
184 /*
185  * Total cache size set by the user.
186  */
187 static unsigned long dm_bufio_cache_size;
188
189 /*
190  * A copy of dm_bufio_cache_size because dm_bufio_cache_size can change
191  * at any time.  If it disagrees, the user has changed cache size.
192  */
193 static unsigned long dm_bufio_cache_size_latch;
194
195 static DEFINE_SPINLOCK(param_spinlock);
196
197 /*
198  * Buffers are freed after this timeout
199  */
200 static unsigned dm_bufio_max_age = DM_BUFIO_DEFAULT_AGE_SECS;
201 static unsigned long dm_bufio_retain_bytes = DM_BUFIO_DEFAULT_RETAIN_BYTES;
202
203 static unsigned long dm_bufio_peak_allocated;
204 static unsigned long dm_bufio_allocated_kmem_cache;
205 static unsigned long dm_bufio_allocated_get_free_pages;
206 static unsigned long dm_bufio_allocated_vmalloc;
207 static unsigned long dm_bufio_current_allocated;
208
209 /*----------------------------------------------------------------*/
210
211 /*
212  * Per-client cache: dm_bufio_cache_size / dm_bufio_client_count
213  */
214 static unsigned long dm_bufio_cache_size_per_client;
215
216 /*
217  * The current number of clients.
218  */
219 static int dm_bufio_client_count;
220
221 /*
222  * The list of all clients.
223  */
224 static LIST_HEAD(dm_bufio_all_clients);
225
226 /*
227  * This mutex protects dm_bufio_cache_size_latch,
228  * dm_bufio_cache_size_per_client and dm_bufio_client_count
229  */
230 static DEFINE_MUTEX(dm_bufio_clients_lock);
231
232 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
233 static void buffer_record_stack(struct dm_buffer *b)
234 {
235         b->stack_trace.nr_entries = 0;
236         b->stack_trace.max_entries = MAX_STACK;
237         b->stack_trace.entries = b->stack_entries;
238         b->stack_trace.skip = 2;
239         save_stack_trace(&b->stack_trace);
240 }
241 #endif
242
243 /*----------------------------------------------------------------
244  * A red/black tree acts as an index for all the buffers.
245  *--------------------------------------------------------------*/
246 static struct dm_buffer *__find(struct dm_bufio_client *c, sector_t block)
247 {
248         struct rb_node *n = c->buffer_tree.rb_node;
249         struct dm_buffer *b;
250
251         while (n) {
252                 b = container_of(n, struct dm_buffer, node);
253
254                 if (b->block == block)
255                         return b;
256
257                 n = (b->block < block) ? n->rb_left : n->rb_right;
258         }
259
260         return NULL;
261 }
262
263 static void __insert(struct dm_bufio_client *c, struct dm_buffer *b)
264 {
265         struct rb_node **new = &c->buffer_tree.rb_node, *parent = NULL;
266         struct dm_buffer *found;
267
268         while (*new) {
269                 found = container_of(*new, struct dm_buffer, node);
270
271                 if (found->block == b->block) {
272                         BUG_ON(found != b);
273                         return;
274                 }
275
276                 parent = *new;
277                 new = (found->block < b->block) ?
278                         &((*new)->rb_left) : &((*new)->rb_right);
279         }
280
281         rb_link_node(&b->node, parent, new);
282         rb_insert_color(&b->node, &c->buffer_tree);
283 }
284
285 static void __remove(struct dm_bufio_client *c, struct dm_buffer *b)
286 {
287         rb_erase(&b->node, &c->buffer_tree);
288 }
289
290 /*----------------------------------------------------------------*/
291
292 static void adjust_total_allocated(unsigned char data_mode, long diff)
293 {
294         static unsigned long * const class_ptr[DATA_MODE_LIMIT] = {
295                 &dm_bufio_allocated_kmem_cache,
296                 &dm_bufio_allocated_get_free_pages,
297                 &dm_bufio_allocated_vmalloc,
298         };
299
300         spin_lock(&param_spinlock);
301
302         *class_ptr[data_mode] += diff;
303
304         dm_bufio_current_allocated += diff;
305
306         if (dm_bufio_current_allocated > dm_bufio_peak_allocated)
307                 dm_bufio_peak_allocated = dm_bufio_current_allocated;
308
309         spin_unlock(&param_spinlock);
310 }
311
312 /*
313  * Change the number of clients and recalculate per-client limit.
314  */
315 static void __cache_size_refresh(void)
316 {
317         BUG_ON(!mutex_is_locked(&dm_bufio_clients_lock));
318         BUG_ON(dm_bufio_client_count < 0);
319
320         dm_bufio_cache_size_latch = READ_ONCE(dm_bufio_cache_size);
321
322         /*
323          * Use default if set to 0 and report the actual cache size used.
324          */
325         if (!dm_bufio_cache_size_latch) {
326                 (void)cmpxchg(&dm_bufio_cache_size, 0,
327                               dm_bufio_default_cache_size);
328                 dm_bufio_cache_size_latch = dm_bufio_default_cache_size;
329         }
330
331         dm_bufio_cache_size_per_client = dm_bufio_cache_size_latch /
332                                          (dm_bufio_client_count ? : 1);
333 }
334
335 /*
336  * Allocating buffer data.
337  *
338  * Small buffers are allocated with kmem_cache, to use space optimally.
339  *
340  * For large buffers, we choose between get_free_pages and vmalloc.
341  * Each has advantages and disadvantages.
342  *
343  * __get_free_pages can randomly fail if the memory is fragmented.
344  * __vmalloc won't randomly fail, but vmalloc space is limited (it may be
345  * as low as 128M) so using it for caching is not appropriate.
346  *
347  * If the allocation may fail we use __get_free_pages. Memory fragmentation
348  * won't have a fatal effect here, but it just causes flushes of some other
349  * buffers and more I/O will be performed. Don't use __get_free_pages if it
350  * always fails (i.e. order >= MAX_ORDER).
351  *
352  * If the allocation shouldn't fail we use __vmalloc. This is only for the
353  * initial reserve allocation, so there's no risk of wasting all vmalloc
354  * space.
355  */
356 static void *alloc_buffer_data(struct dm_bufio_client *c, gfp_t gfp_mask,
357                                unsigned char *data_mode)
358 {
359         if (unlikely(c->slab_cache != NULL)) {
360                 *data_mode = DATA_MODE_SLAB;
361                 return kmem_cache_alloc(c->slab_cache, gfp_mask);
362         }
363
364         if (c->block_size <= KMALLOC_MAX_SIZE &&
365             gfp_mask & __GFP_NORETRY) {
366                 *data_mode = DATA_MODE_GET_FREE_PAGES;
367                 return (void *)__get_free_pages(gfp_mask,
368                                                 c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT));
369         }
370
371         *data_mode = DATA_MODE_VMALLOC;
372
373         /*
374          * __vmalloc allocates the data pages and auxiliary structures with
375          * gfp_flags that were specified, but pagetables are always allocated
376          * with GFP_KERNEL, no matter what was specified as gfp_mask.
377          *
378          * Consequently, we must set per-process flag PF_MEMALLOC_NOIO so that
379          * all allocations done by this process (including pagetables) are done
380          * as if GFP_NOIO was specified.
381          */
382         if (gfp_mask & __GFP_NORETRY) {
383                 unsigned noio_flag = memalloc_noio_save();
384                 void *ptr = __vmalloc(c->block_size, gfp_mask, PAGE_KERNEL);
385
386                 memalloc_noio_restore(noio_flag);
387                 return ptr;
388         }
389
390         return __vmalloc(c->block_size, gfp_mask, PAGE_KERNEL);
391 }
392
393 /*
394  * Free buffer's data.
395  */
396 static void free_buffer_data(struct dm_bufio_client *c,
397                              void *data, unsigned char data_mode)
398 {
399         switch (data_mode) {
400         case DATA_MODE_SLAB:
401                 kmem_cache_free(c->slab_cache, data);
402                 break;
403
404         case DATA_MODE_GET_FREE_PAGES:
405                 free_pages((unsigned long)data,
406                            c->sectors_per_block_bits - (PAGE_SHIFT - SECTOR_SHIFT));
407                 break;
408
409         case DATA_MODE_VMALLOC:
410                 vfree(data);
411                 break;
412
413         default:
414                 DMCRIT("dm_bufio_free_buffer_data: bad data mode: %d",
415                        data_mode);
416                 BUG();
417         }
418 }
419
420 /*
421  * Allocate buffer and its data.
422  */
423 static struct dm_buffer *alloc_buffer(struct dm_bufio_client *c, gfp_t gfp_mask)
424 {
425         struct dm_buffer *b = kmem_cache_alloc(c->slab_buffer, gfp_mask);
426
427         if (!b)
428                 return NULL;
429
430         b->c = c;
431
432         b->data = alloc_buffer_data(c, gfp_mask, &b->data_mode);
433         if (!b->data) {
434                 kmem_cache_free(c->slab_buffer, b);
435                 return NULL;
436         }
437
438         adjust_total_allocated(b->data_mode, (long)c->block_size);
439
440 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
441         memset(&b->stack_trace, 0, sizeof(b->stack_trace));
442 #endif
443         return b;
444 }
445
446 /*
447  * Free buffer and its data.
448  */
449 static void free_buffer(struct dm_buffer *b)
450 {
451         struct dm_bufio_client *c = b->c;
452
453         adjust_total_allocated(b->data_mode, -(long)c->block_size);
454
455         free_buffer_data(c, b->data, b->data_mode);
456         kmem_cache_free(c->slab_buffer, b);
457 }
458
459 /*
460  * Link buffer to the hash list and clean or dirty queue.
461  */
462 static void __link_buffer(struct dm_buffer *b, sector_t block, int dirty)
463 {
464         struct dm_bufio_client *c = b->c;
465
466         c->n_buffers[dirty]++;
467         b->block = block;
468         b->list_mode = dirty;
469         list_add(&b->lru_list, &c->lru[dirty]);
470         __insert(b->c, b);
471         b->last_accessed = jiffies;
472 }
473
474 /*
475  * Unlink buffer from the hash list and dirty or clean queue.
476  */
477 static void __unlink_buffer(struct dm_buffer *b)
478 {
479         struct dm_bufio_client *c = b->c;
480
481         BUG_ON(!c->n_buffers[b->list_mode]);
482
483         c->n_buffers[b->list_mode]--;
484         __remove(b->c, b);
485         list_del(&b->lru_list);
486 }
487
488 /*
489  * Place the buffer to the head of dirty or clean LRU queue.
490  */
491 static void __relink_lru(struct dm_buffer *b, int dirty)
492 {
493         struct dm_bufio_client *c = b->c;
494
495         BUG_ON(!c->n_buffers[b->list_mode]);
496
497         c->n_buffers[b->list_mode]--;
498         c->n_buffers[dirty]++;
499         b->list_mode = dirty;
500         list_move(&b->lru_list, &c->lru[dirty]);
501         b->last_accessed = jiffies;
502 }
503
504 /*----------------------------------------------------------------
505  * Submit I/O on the buffer.
506  *
507  * Bio interface is faster but it has some problems:
508  *      the vector list is limited (increasing this limit increases
509  *      memory-consumption per buffer, so it is not viable);
510  *
511  *      the memory must be direct-mapped, not vmalloced;
512  *
513  * If the buffer is small enough (up to DM_BUFIO_INLINE_VECS pages) and
514  * it is not vmalloced, try using the bio interface.
515  *
516  * If the buffer is big, if it is vmalloced or if the underlying device
517  * rejects the bio because it is too large, use dm-io layer to do the I/O.
518  * The dm-io layer splits the I/O into multiple requests, avoiding the above
519  * shortcomings.
520  *--------------------------------------------------------------*/
521
522 /*
523  * dm-io completion routine. It just calls b->bio.bi_end_io, pretending
524  * that the request was handled directly with bio interface.
525  */
526 static void dmio_complete(unsigned long error, void *context)
527 {
528         struct dm_buffer *b = context;
529
530         b->end_io(b, unlikely(error != 0) ? BLK_STS_IOERR : 0);
531 }
532
533 static void use_dmio(struct dm_buffer *b, int rw, sector_t sector,
534                      unsigned n_sectors, unsigned offset)
535 {
536         int r;
537         struct dm_io_request io_req = {
538                 .bi_op = rw,
539                 .bi_op_flags = 0,
540                 .notify.fn = dmio_complete,
541                 .notify.context = b,
542                 .client = b->c->dm_io,
543         };
544         struct dm_io_region region = {
545                 .bdev = b->c->bdev,
546                 .sector = sector,
547                 .count = n_sectors,
548         };
549
550         if (b->data_mode != DATA_MODE_VMALLOC) {
551                 io_req.mem.type = DM_IO_KMEM;
552                 io_req.mem.ptr.addr = (char *)b->data + offset;
553         } else {
554                 io_req.mem.type = DM_IO_VMA;
555                 io_req.mem.ptr.vma = (char *)b->data + offset;
556         }
557
558         r = dm_io(&io_req, 1, &region, NULL);
559         if (unlikely(r))
560                 b->end_io(b, errno_to_blk_status(r));
561 }
562
563 static void bio_complete(struct bio *bio)
564 {
565         struct dm_buffer *b = bio->bi_private;
566         blk_status_t status = bio->bi_status;
567         bio_put(bio);
568         b->end_io(b, status);
569 }
570
571 static void use_bio(struct dm_buffer *b, int rw, sector_t sector,
572                     unsigned n_sectors, unsigned offset)
573 {
574         struct bio *bio;
575         char *ptr;
576         unsigned vec_size, len;
577
578         vec_size = b->c->block_size >> PAGE_SHIFT;
579         if (unlikely(b->c->sectors_per_block_bits < PAGE_SHIFT - SECTOR_SHIFT))
580                 vec_size += 2;
581
582         bio = bio_kmalloc(GFP_NOWAIT | __GFP_NORETRY | __GFP_NOWARN, vec_size);
583         if (!bio) {
584 dmio:
585                 use_dmio(b, rw, sector, n_sectors, offset);
586                 return;
587         }
588
589         bio->bi_iter.bi_sector = sector;
590         bio_set_dev(bio, b->c->bdev);
591         bio_set_op_attrs(bio, rw, 0);
592         bio->bi_end_io = bio_complete;
593         bio->bi_private = b;
594
595         ptr = (char *)b->data + offset;
596         len = n_sectors << SECTOR_SHIFT;
597
598         do {
599                 unsigned this_step = min((unsigned)(PAGE_SIZE - offset_in_page(ptr)), len);
600                 if (!bio_add_page(bio, virt_to_page(ptr), this_step,
601                                   offset_in_page(ptr))) {
602                         bio_put(bio);
603                         goto dmio;
604                 }
605
606                 len -= this_step;
607                 ptr += this_step;
608         } while (len > 0);
609
610         submit_bio(bio);
611 }
612
613 static void submit_io(struct dm_buffer *b, int rw, void (*end_io)(struct dm_buffer *, blk_status_t))
614 {
615         unsigned n_sectors;
616         sector_t sector;
617         unsigned offset, end;
618
619         b->end_io = end_io;
620
621         if (likely(b->c->sectors_per_block_bits >= 0))
622                 sector = b->block << b->c->sectors_per_block_bits;
623         else
624                 sector = b->block * (b->c->block_size >> SECTOR_SHIFT);
625         sector += b->c->start;
626
627         if (rw != REQ_OP_WRITE) {
628                 n_sectors = b->c->block_size >> SECTOR_SHIFT;
629                 offset = 0;
630         } else {
631                 if (b->c->write_callback)
632                         b->c->write_callback(b);
633                 offset = b->write_start;
634                 end = b->write_end;
635                 offset &= -DM_BUFIO_WRITE_ALIGN;
636                 end += DM_BUFIO_WRITE_ALIGN - 1;
637                 end &= -DM_BUFIO_WRITE_ALIGN;
638                 if (unlikely(end > b->c->block_size))
639                         end = b->c->block_size;
640
641                 sector += offset >> SECTOR_SHIFT;
642                 n_sectors = (end - offset) >> SECTOR_SHIFT;
643         }
644
645         if (b->data_mode != DATA_MODE_VMALLOC)
646                 use_bio(b, rw, sector, n_sectors, offset);
647         else
648                 use_dmio(b, rw, sector, n_sectors, offset);
649 }
650
651 /*----------------------------------------------------------------
652  * Writing dirty buffers
653  *--------------------------------------------------------------*/
654
655 /*
656  * The endio routine for write.
657  *
658  * Set the error, clear B_WRITING bit and wake anyone who was waiting on
659  * it.
660  */
661 static void write_endio(struct dm_buffer *b, blk_status_t status)
662 {
663         b->write_error = status;
664         if (unlikely(status)) {
665                 struct dm_bufio_client *c = b->c;
666
667                 (void)cmpxchg(&c->async_write_error, 0,
668                                 blk_status_to_errno(status));
669         }
670
671         BUG_ON(!test_bit(B_WRITING, &b->state));
672
673         smp_mb__before_atomic();
674         clear_bit(B_WRITING, &b->state);
675         smp_mb__after_atomic();
676
677         wake_up_bit(&b->state, B_WRITING);
678 }
679
680 /*
681  * Initiate a write on a dirty buffer, but don't wait for it.
682  *
683  * - If the buffer is not dirty, exit.
684  * - If there some previous write going on, wait for it to finish (we can't
685  *   have two writes on the same buffer simultaneously).
686  * - Submit our write and don't wait on it. We set B_WRITING indicating
687  *   that there is a write in progress.
688  */
689 static void __write_dirty_buffer(struct dm_buffer *b,
690                                  struct list_head *write_list)
691 {
692         if (!test_bit(B_DIRTY, &b->state))
693                 return;
694
695         clear_bit(B_DIRTY, &b->state);
696         wait_on_bit_lock_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
697
698         b->write_start = b->dirty_start;
699         b->write_end = b->dirty_end;
700
701         if (!write_list)
702                 submit_io(b, REQ_OP_WRITE, write_endio);
703         else
704                 list_add_tail(&b->write_list, write_list);
705 }
706
707 static void __flush_write_list(struct list_head *write_list)
708 {
709         struct blk_plug plug;
710         blk_start_plug(&plug);
711         while (!list_empty(write_list)) {
712                 struct dm_buffer *b =
713                         list_entry(write_list->next, struct dm_buffer, write_list);
714                 list_del(&b->write_list);
715                 submit_io(b, REQ_OP_WRITE, write_endio);
716                 cond_resched();
717         }
718         blk_finish_plug(&plug);
719 }
720
721 /*
722  * Wait until any activity on the buffer finishes.  Possibly write the
723  * buffer if it is dirty.  When this function finishes, there is no I/O
724  * running on the buffer and the buffer is not dirty.
725  */
726 static void __make_buffer_clean(struct dm_buffer *b)
727 {
728         BUG_ON(b->hold_count);
729
730         if (!b->state)  /* fast case */
731                 return;
732
733         wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
734         __write_dirty_buffer(b, NULL);
735         wait_on_bit_io(&b->state, B_WRITING, TASK_UNINTERRUPTIBLE);
736 }
737
738 /*
739  * Find some buffer that is not held by anybody, clean it, unlink it and
740  * return it.
741  */
742 static struct dm_buffer *__get_unclaimed_buffer(struct dm_bufio_client *c)
743 {
744         struct dm_buffer *b;
745
746         list_for_each_entry_reverse(b, &c->lru[LIST_CLEAN], lru_list) {
747                 BUG_ON(test_bit(B_WRITING, &b->state));
748                 BUG_ON(test_bit(B_DIRTY, &b->state));
749
750                 if (!b->hold_count) {
751                         __make_buffer_clean(b);
752                         __unlink_buffer(b);
753                         return b;
754                 }
755                 cond_resched();
756         }
757
758         list_for_each_entry_reverse(b, &c->lru[LIST_DIRTY], lru_list) {
759                 BUG_ON(test_bit(B_READING, &b->state));
760
761                 if (!b->hold_count) {
762                         __make_buffer_clean(b);
763                         __unlink_buffer(b);
764                         return b;
765                 }
766                 cond_resched();
767         }
768
769         return NULL;
770 }
771
772 /*
773  * Wait until some other threads free some buffer or release hold count on
774  * some buffer.
775  *
776  * This function is entered with c->lock held, drops it and regains it
777  * before exiting.
778  */
779 static void __wait_for_free_buffer(struct dm_bufio_client *c)
780 {
781         DECLARE_WAITQUEUE(wait, current);
782
783         add_wait_queue(&c->free_buffer_wait, &wait);
784         set_current_state(TASK_UNINTERRUPTIBLE);
785         dm_bufio_unlock(c);
786
787         io_schedule();
788
789         remove_wait_queue(&c->free_buffer_wait, &wait);
790
791         dm_bufio_lock(c);
792 }
793
794 enum new_flag {
795         NF_FRESH = 0,
796         NF_READ = 1,
797         NF_GET = 2,
798         NF_PREFETCH = 3
799 };
800
801 /*
802  * Allocate a new buffer. If the allocation is not possible, wait until
803  * some other thread frees a buffer.
804  *
805  * May drop the lock and regain it.
806  */
807 static struct dm_buffer *__alloc_buffer_wait_no_callback(struct dm_bufio_client *c, enum new_flag nf)
808 {
809         struct dm_buffer *b;
810         bool tried_noio_alloc = false;
811
812         /*
813          * dm-bufio is resistant to allocation failures (it just keeps
814          * one buffer reserved in cases all the allocations fail).
815          * So set flags to not try too hard:
816          *      GFP_NOWAIT: don't wait; if we need to sleep we'll release our
817          *                  mutex and wait ourselves.
818          *      __GFP_NORETRY: don't retry and rather return failure
819          *      __GFP_NOMEMALLOC: don't use emergency reserves
820          *      __GFP_NOWARN: don't print a warning in case of failure
821          *
822          * For debugging, if we set the cache size to 1, no new buffers will
823          * be allocated.
824          */
825         while (1) {
826                 if (dm_bufio_cache_size_latch != 1) {
827                         b = alloc_buffer(c, GFP_NOWAIT | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
828                         if (b)
829                                 return b;
830                 }
831
832                 if (nf == NF_PREFETCH)
833                         return NULL;
834
835                 if (dm_bufio_cache_size_latch != 1 && !tried_noio_alloc) {
836                         dm_bufio_unlock(c);
837                         b = alloc_buffer(c, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
838                         dm_bufio_lock(c);
839                         if (b)
840                                 return b;
841                         tried_noio_alloc = true;
842                 }
843
844                 if (!list_empty(&c->reserved_buffers)) {
845                         b = list_entry(c->reserved_buffers.next,
846                                        struct dm_buffer, lru_list);
847                         list_del(&b->lru_list);
848                         c->need_reserved_buffers++;
849
850                         return b;
851                 }
852
853                 b = __get_unclaimed_buffer(c);
854                 if (b)
855                         return b;
856
857                 __wait_for_free_buffer(c);
858         }
859 }
860
861 static struct dm_buffer *__alloc_buffer_wait(struct dm_bufio_client *c, enum new_flag nf)
862 {
863         struct dm_buffer *b = __alloc_buffer_wait_no_callback(c, nf);
864
865         if (!b)
866                 return NULL;
867
868         if (c->alloc_callback)
869                 c->alloc_callback(b);
870
871         return b;
872 }
873
874 /*
875  * Free a buffer and wake other threads waiting for free buffers.
876  */
877 static void __free_buffer_wake(struct dm_buffer *b)
878 {
879         struct dm_bufio_client *c = b->c;
880
881         if (!c->need_reserved_buffers)
882                 free_buffer(b);
883         else {
884                 list_add(&b->lru_list, &c->reserved_buffers);
885                 c->need_reserved_buffers--;
886         }
887
888         wake_up(&c->free_buffer_wait);
889 }
890
891 static void __write_dirty_buffers_async(struct dm_bufio_client *c, int no_wait,
892                                         struct list_head *write_list)
893 {
894         struct dm_buffer *b, *tmp;
895
896         list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) {
897                 BUG_ON(test_bit(B_READING, &b->state));
898
899                 if (!test_bit(B_DIRTY, &b->state) &&
900                     !test_bit(B_WRITING, &b->state)) {
901                         __relink_lru(b, LIST_CLEAN);
902                         continue;
903                 }
904
905                 if (no_wait && test_bit(B_WRITING, &b->state))
906                         return;
907
908                 __write_dirty_buffer(b, write_list);
909                 cond_resched();
910         }
911 }
912
913 /*
914  * Get writeback threshold and buffer limit for a given client.
915  */
916 static void __get_memory_limit(struct dm_bufio_client *c,
917                                unsigned long *threshold_buffers,
918                                unsigned long *limit_buffers)
919 {
920         unsigned long buffers;
921
922         if (unlikely(READ_ONCE(dm_bufio_cache_size) != dm_bufio_cache_size_latch)) {
923                 if (mutex_trylock(&dm_bufio_clients_lock)) {
924                         __cache_size_refresh();
925                         mutex_unlock(&dm_bufio_clients_lock);
926                 }
927         }
928
929         buffers = dm_bufio_cache_size_per_client;
930         if (likely(c->sectors_per_block_bits >= 0))
931                 buffers >>= c->sectors_per_block_bits + SECTOR_SHIFT;
932         else
933                 buffers /= c->block_size;
934
935         if (buffers < c->minimum_buffers)
936                 buffers = c->minimum_buffers;
937
938         *limit_buffers = buffers;
939         *threshold_buffers = mult_frac(buffers,
940                                        DM_BUFIO_WRITEBACK_PERCENT, 100);
941 }
942
943 /*
944  * Check if we're over watermark.
945  * If we are over threshold_buffers, start freeing buffers.
946  * If we're over "limit_buffers", block until we get under the limit.
947  */
948 static void __check_watermark(struct dm_bufio_client *c,
949                               struct list_head *write_list)
950 {
951         unsigned long threshold_buffers, limit_buffers;
952
953         __get_memory_limit(c, &threshold_buffers, &limit_buffers);
954
955         while (c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY] >
956                limit_buffers) {
957
958                 struct dm_buffer *b = __get_unclaimed_buffer(c);
959
960                 if (!b)
961                         return;
962
963                 __free_buffer_wake(b);
964                 cond_resched();
965         }
966
967         if (c->n_buffers[LIST_DIRTY] > threshold_buffers)
968                 __write_dirty_buffers_async(c, 1, write_list);
969 }
970
971 /*----------------------------------------------------------------
972  * Getting a buffer
973  *--------------------------------------------------------------*/
974
975 static struct dm_buffer *__bufio_new(struct dm_bufio_client *c, sector_t block,
976                                      enum new_flag nf, int *need_submit,
977                                      struct list_head *write_list)
978 {
979         struct dm_buffer *b, *new_b = NULL;
980
981         *need_submit = 0;
982
983         b = __find(c, block);
984         if (b)
985                 goto found_buffer;
986
987         if (nf == NF_GET)
988                 return NULL;
989
990         new_b = __alloc_buffer_wait(c, nf);
991         if (!new_b)
992                 return NULL;
993
994         /*
995          * We've had a period where the mutex was unlocked, so need to
996          * recheck the hash table.
997          */
998         b = __find(c, block);
999         if (b) {
1000                 __free_buffer_wake(new_b);
1001                 goto found_buffer;
1002         }
1003
1004         __check_watermark(c, write_list);
1005
1006         b = new_b;
1007         b->hold_count = 1;
1008         b->read_error = 0;
1009         b->write_error = 0;
1010         __link_buffer(b, block, LIST_CLEAN);
1011
1012         if (nf == NF_FRESH) {
1013                 b->state = 0;
1014                 return b;
1015         }
1016
1017         b->state = 1 << B_READING;
1018         *need_submit = 1;
1019
1020         return b;
1021
1022 found_buffer:
1023         if (nf == NF_PREFETCH)
1024                 return NULL;
1025         /*
1026          * Note: it is essential that we don't wait for the buffer to be
1027          * read if dm_bufio_get function is used. Both dm_bufio_get and
1028          * dm_bufio_prefetch can be used in the driver request routine.
1029          * If the user called both dm_bufio_prefetch and dm_bufio_get on
1030          * the same buffer, it would deadlock if we waited.
1031          */
1032         if (nf == NF_GET && unlikely(test_bit(B_READING, &b->state)))
1033                 return NULL;
1034
1035         b->hold_count++;
1036         __relink_lru(b, test_bit(B_DIRTY, &b->state) ||
1037                      test_bit(B_WRITING, &b->state));
1038         return b;
1039 }
1040
1041 /*
1042  * The endio routine for reading: set the error, clear the bit and wake up
1043  * anyone waiting on the buffer.
1044  */
1045 static void read_endio(struct dm_buffer *b, blk_status_t status)
1046 {
1047         b->read_error = status;
1048
1049         BUG_ON(!test_bit(B_READING, &b->state));
1050
1051         smp_mb__before_atomic();
1052         clear_bit(B_READING, &b->state);
1053         smp_mb__after_atomic();
1054
1055         wake_up_bit(&b->state, B_READING);
1056 }
1057
1058 /*
1059  * A common routine for dm_bufio_new and dm_bufio_read.  Operation of these
1060  * functions is similar except that dm_bufio_new doesn't read the
1061  * buffer from the disk (assuming that the caller overwrites all the data
1062  * and uses dm_bufio_mark_buffer_dirty to write new data back).
1063  */
1064 static void *new_read(struct dm_bufio_client *c, sector_t block,
1065                       enum new_flag nf, struct dm_buffer **bp)
1066 {
1067         int need_submit;
1068         struct dm_buffer *b;
1069
1070         LIST_HEAD(write_list);
1071
1072         dm_bufio_lock(c);
1073         b = __bufio_new(c, block, nf, &need_submit, &write_list);
1074 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
1075         if (b && b->hold_count == 1)
1076                 buffer_record_stack(b);
1077 #endif
1078         dm_bufio_unlock(c);
1079
1080         __flush_write_list(&write_list);
1081
1082         if (!b)
1083                 return NULL;
1084
1085         if (need_submit)
1086                 submit_io(b, REQ_OP_READ, read_endio);
1087
1088         wait_on_bit_io(&b->state, B_READING, TASK_UNINTERRUPTIBLE);
1089
1090         if (b->read_error) {
1091                 int error = blk_status_to_errno(b->read_error);
1092
1093                 dm_bufio_release(b);
1094
1095                 return ERR_PTR(error);
1096         }
1097
1098         *bp = b;
1099
1100         return b->data;
1101 }
1102
1103 void *dm_bufio_get(struct dm_bufio_client *c, sector_t block,
1104                    struct dm_buffer **bp)
1105 {
1106         return new_read(c, block, NF_GET, bp);
1107 }
1108 EXPORT_SYMBOL_GPL(dm_bufio_get);
1109
1110 void *dm_bufio_read(struct dm_bufio_client *c, sector_t block,
1111                     struct dm_buffer **bp)
1112 {
1113         BUG_ON(dm_bufio_in_request());
1114
1115         return new_read(c, block, NF_READ, bp);
1116 }
1117 EXPORT_SYMBOL_GPL(dm_bufio_read);
1118
1119 void *dm_bufio_new(struct dm_bufio_client *c, sector_t block,
1120                    struct dm_buffer **bp)
1121 {
1122         BUG_ON(dm_bufio_in_request());
1123
1124         return new_read(c, block, NF_FRESH, bp);
1125 }
1126 EXPORT_SYMBOL_GPL(dm_bufio_new);
1127
1128 void dm_bufio_prefetch(struct dm_bufio_client *c,
1129                        sector_t block, unsigned n_blocks)
1130 {
1131         struct blk_plug plug;
1132
1133         LIST_HEAD(write_list);
1134
1135         BUG_ON(dm_bufio_in_request());
1136
1137         blk_start_plug(&plug);
1138         dm_bufio_lock(c);
1139
1140         for (; n_blocks--; block++) {
1141                 int need_submit;
1142                 struct dm_buffer *b;
1143                 b = __bufio_new(c, block, NF_PREFETCH, &need_submit,
1144                                 &write_list);
1145                 if (unlikely(!list_empty(&write_list))) {
1146                         dm_bufio_unlock(c);
1147                         blk_finish_plug(&plug);
1148                         __flush_write_list(&write_list);
1149                         blk_start_plug(&plug);
1150                         dm_bufio_lock(c);
1151                 }
1152                 if (unlikely(b != NULL)) {
1153                         dm_bufio_unlock(c);
1154
1155                         if (need_submit)
1156                                 submit_io(b, REQ_OP_READ, read_endio);
1157                         dm_bufio_release(b);
1158
1159                         cond_resched();
1160
1161                         if (!n_blocks)
1162                                 goto flush_plug;
1163                         dm_bufio_lock(c);
1164                 }
1165         }
1166
1167         dm_bufio_unlock(c);
1168
1169 flush_plug:
1170         blk_finish_plug(&plug);
1171 }
1172 EXPORT_SYMBOL_GPL(dm_bufio_prefetch);
1173
1174 void dm_bufio_release(struct dm_buffer *b)
1175 {
1176         struct dm_bufio_client *c = b->c;
1177
1178         dm_bufio_lock(c);
1179
1180         BUG_ON(!b->hold_count);
1181
1182         b->hold_count--;
1183         if (!b->hold_count) {
1184                 wake_up(&c->free_buffer_wait);
1185
1186                 /*
1187                  * If there were errors on the buffer, and the buffer is not
1188                  * to be written, free the buffer. There is no point in caching
1189                  * invalid buffer.
1190                  */
1191                 if ((b->read_error || b->write_error) &&
1192                     !test_bit(B_READING, &b->state) &&
1193                     !test_bit(B_WRITING, &b->state) &&
1194                     !test_bit(B_DIRTY, &b->state)) {
1195                         __unlink_buffer(b);
1196                         __free_buffer_wake(b);
1197                 }
1198         }
1199
1200         dm_bufio_unlock(c);
1201 }
1202 EXPORT_SYMBOL_GPL(dm_bufio_release);
1203
1204 void dm_bufio_mark_partial_buffer_dirty(struct dm_buffer *b,
1205                                         unsigned start, unsigned end)
1206 {
1207         struct dm_bufio_client *c = b->c;
1208
1209         BUG_ON(start >= end);
1210         BUG_ON(end > b->c->block_size);
1211
1212         dm_bufio_lock(c);
1213
1214         BUG_ON(test_bit(B_READING, &b->state));
1215
1216         if (!test_and_set_bit(B_DIRTY, &b->state)) {
1217                 b->dirty_start = start;
1218                 b->dirty_end = end;
1219                 __relink_lru(b, LIST_DIRTY);
1220         } else {
1221                 if (start < b->dirty_start)
1222                         b->dirty_start = start;
1223                 if (end > b->dirty_end)
1224                         b->dirty_end = end;
1225         }
1226
1227         dm_bufio_unlock(c);
1228 }
1229 EXPORT_SYMBOL_GPL(dm_bufio_mark_partial_buffer_dirty);
1230
1231 void dm_bufio_mark_buffer_dirty(struct dm_buffer *b)
1232 {
1233         dm_bufio_mark_partial_buffer_dirty(b, 0, b->c->block_size);
1234 }
1235 EXPORT_SYMBOL_GPL(dm_bufio_mark_buffer_dirty);
1236
1237 void dm_bufio_write_dirty_buffers_async(struct dm_bufio_client *c)
1238 {
1239         LIST_HEAD(write_list);
1240
1241         BUG_ON(dm_bufio_in_request());
1242
1243         dm_bufio_lock(c);
1244         __write_dirty_buffers_async(c, 0, &write_list);
1245         dm_bufio_unlock(c);
1246         __flush_write_list(&write_list);
1247 }
1248 EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers_async);
1249
1250 /*
1251  * For performance, it is essential that the buffers are written asynchronously
1252  * and simultaneously (so that the block layer can merge the writes) and then
1253  * waited upon.
1254  *
1255  * Finally, we flush hardware disk cache.
1256  */
1257 int dm_bufio_write_dirty_buffers(struct dm_bufio_client *c)
1258 {
1259         int a, f;
1260         unsigned long buffers_processed = 0;
1261         struct dm_buffer *b, *tmp;
1262
1263         LIST_HEAD(write_list);
1264
1265         dm_bufio_lock(c);
1266         __write_dirty_buffers_async(c, 0, &write_list);
1267         dm_bufio_unlock(c);
1268         __flush_write_list(&write_list);
1269         dm_bufio_lock(c);
1270
1271 again:
1272         list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_DIRTY], lru_list) {
1273                 int dropped_lock = 0;
1274
1275                 if (buffers_processed < c->n_buffers[LIST_DIRTY])
1276                         buffers_processed++;
1277
1278                 BUG_ON(test_bit(B_READING, &b->state));
1279
1280                 if (test_bit(B_WRITING, &b->state)) {
1281                         if (buffers_processed < c->n_buffers[LIST_DIRTY]) {
1282                                 dropped_lock = 1;
1283                                 b->hold_count++;
1284                                 dm_bufio_unlock(c);
1285                                 wait_on_bit_io(&b->state, B_WRITING,
1286                                                TASK_UNINTERRUPTIBLE);
1287                                 dm_bufio_lock(c);
1288                                 b->hold_count--;
1289                         } else
1290                                 wait_on_bit_io(&b->state, B_WRITING,
1291                                                TASK_UNINTERRUPTIBLE);
1292                 }
1293
1294                 if (!test_bit(B_DIRTY, &b->state) &&
1295                     !test_bit(B_WRITING, &b->state))
1296                         __relink_lru(b, LIST_CLEAN);
1297
1298                 cond_resched();
1299
1300                 /*
1301                  * If we dropped the lock, the list is no longer consistent,
1302                  * so we must restart the search.
1303                  *
1304                  * In the most common case, the buffer just processed is
1305                  * relinked to the clean list, so we won't loop scanning the
1306                  * same buffer again and again.
1307                  *
1308                  * This may livelock if there is another thread simultaneously
1309                  * dirtying buffers, so we count the number of buffers walked
1310                  * and if it exceeds the total number of buffers, it means that
1311                  * someone is doing some writes simultaneously with us.  In
1312                  * this case, stop, dropping the lock.
1313                  */
1314                 if (dropped_lock)
1315                         goto again;
1316         }
1317         wake_up(&c->free_buffer_wait);
1318         dm_bufio_unlock(c);
1319
1320         a = xchg(&c->async_write_error, 0);
1321         f = dm_bufio_issue_flush(c);
1322         if (a)
1323                 return a;
1324
1325         return f;
1326 }
1327 EXPORT_SYMBOL_GPL(dm_bufio_write_dirty_buffers);
1328
1329 /*
1330  * Use dm-io to send and empty barrier flush the device.
1331  */
1332 int dm_bufio_issue_flush(struct dm_bufio_client *c)
1333 {
1334         struct dm_io_request io_req = {
1335                 .bi_op = REQ_OP_WRITE,
1336                 .bi_op_flags = REQ_PREFLUSH | REQ_SYNC,
1337                 .mem.type = DM_IO_KMEM,
1338                 .mem.ptr.addr = NULL,
1339                 .client = c->dm_io,
1340         };
1341         struct dm_io_region io_reg = {
1342                 .bdev = c->bdev,
1343                 .sector = 0,
1344                 .count = 0,
1345         };
1346
1347         BUG_ON(dm_bufio_in_request());
1348
1349         return dm_io(&io_req, 1, &io_reg, NULL);
1350 }
1351 EXPORT_SYMBOL_GPL(dm_bufio_issue_flush);
1352
1353 /*
1354  * We first delete any other buffer that may be at that new location.
1355  *
1356  * Then, we write the buffer to the original location if it was dirty.
1357  *
1358  * Then, if we are the only one who is holding the buffer, relink the buffer
1359  * in the hash queue for the new location.
1360  *
1361  * If there was someone else holding the buffer, we write it to the new
1362  * location but not relink it, because that other user needs to have the buffer
1363  * at the same place.
1364  */
1365 void dm_bufio_release_move(struct dm_buffer *b, sector_t new_block)
1366 {
1367         struct dm_bufio_client *c = b->c;
1368         struct dm_buffer *new;
1369
1370         BUG_ON(dm_bufio_in_request());
1371
1372         dm_bufio_lock(c);
1373
1374 retry:
1375         new = __find(c, new_block);
1376         if (new) {
1377                 if (new->hold_count) {
1378                         __wait_for_free_buffer(c);
1379                         goto retry;
1380                 }
1381
1382                 /*
1383                  * FIXME: Is there any point waiting for a write that's going
1384                  * to be overwritten in a bit?
1385                  */
1386                 __make_buffer_clean(new);
1387                 __unlink_buffer(new);
1388                 __free_buffer_wake(new);
1389         }
1390
1391         BUG_ON(!b->hold_count);
1392         BUG_ON(test_bit(B_READING, &b->state));
1393
1394         __write_dirty_buffer(b, NULL);
1395         if (b->hold_count == 1) {
1396                 wait_on_bit_io(&b->state, B_WRITING,
1397                                TASK_UNINTERRUPTIBLE);
1398                 set_bit(B_DIRTY, &b->state);
1399                 b->dirty_start = 0;
1400                 b->dirty_end = c->block_size;
1401                 __unlink_buffer(b);
1402                 __link_buffer(b, new_block, LIST_DIRTY);
1403         } else {
1404                 sector_t old_block;
1405                 wait_on_bit_lock_io(&b->state, B_WRITING,
1406                                     TASK_UNINTERRUPTIBLE);
1407                 /*
1408                  * Relink buffer to "new_block" so that write_callback
1409                  * sees "new_block" as a block number.
1410                  * After the write, link the buffer back to old_block.
1411                  * All this must be done in bufio lock, so that block number
1412                  * change isn't visible to other threads.
1413                  */
1414                 old_block = b->block;
1415                 __unlink_buffer(b);
1416                 __link_buffer(b, new_block, b->list_mode);
1417                 submit_io(b, REQ_OP_WRITE, write_endio);
1418                 wait_on_bit_io(&b->state, B_WRITING,
1419                                TASK_UNINTERRUPTIBLE);
1420                 __unlink_buffer(b);
1421                 __link_buffer(b, old_block, b->list_mode);
1422         }
1423
1424         dm_bufio_unlock(c);
1425         dm_bufio_release(b);
1426 }
1427 EXPORT_SYMBOL_GPL(dm_bufio_release_move);
1428
1429 /*
1430  * Free the given buffer.
1431  *
1432  * This is just a hint, if the buffer is in use or dirty, this function
1433  * does nothing.
1434  */
1435 void dm_bufio_forget(struct dm_bufio_client *c, sector_t block)
1436 {
1437         struct dm_buffer *b;
1438
1439         dm_bufio_lock(c);
1440
1441         b = __find(c, block);
1442         if (b && likely(!b->hold_count) && likely(!b->state)) {
1443                 __unlink_buffer(b);
1444                 __free_buffer_wake(b);
1445         }
1446
1447         dm_bufio_unlock(c);
1448 }
1449 EXPORT_SYMBOL_GPL(dm_bufio_forget);
1450
1451 void dm_bufio_set_minimum_buffers(struct dm_bufio_client *c, unsigned n)
1452 {
1453         c->minimum_buffers = n;
1454 }
1455 EXPORT_SYMBOL_GPL(dm_bufio_set_minimum_buffers);
1456
1457 unsigned dm_bufio_get_block_size(struct dm_bufio_client *c)
1458 {
1459         return c->block_size;
1460 }
1461 EXPORT_SYMBOL_GPL(dm_bufio_get_block_size);
1462
1463 sector_t dm_bufio_get_device_size(struct dm_bufio_client *c)
1464 {
1465         sector_t s = i_size_read(c->bdev->bd_inode) >> SECTOR_SHIFT;
1466         if (s >= c->start)
1467                 s -= c->start;
1468         else
1469                 s = 0;
1470         if (likely(c->sectors_per_block_bits >= 0))
1471                 s >>= c->sectors_per_block_bits;
1472         else
1473                 sector_div(s, c->block_size >> SECTOR_SHIFT);
1474         return s;
1475 }
1476 EXPORT_SYMBOL_GPL(dm_bufio_get_device_size);
1477
1478 struct dm_io_client *dm_bufio_get_dm_io_client(struct dm_bufio_client *c)
1479 {
1480         return c->dm_io;
1481 }
1482 EXPORT_SYMBOL_GPL(dm_bufio_get_dm_io_client);
1483
1484 sector_t dm_bufio_get_block_number(struct dm_buffer *b)
1485 {
1486         return b->block;
1487 }
1488 EXPORT_SYMBOL_GPL(dm_bufio_get_block_number);
1489
1490 void *dm_bufio_get_block_data(struct dm_buffer *b)
1491 {
1492         return b->data;
1493 }
1494 EXPORT_SYMBOL_GPL(dm_bufio_get_block_data);
1495
1496 void *dm_bufio_get_aux_data(struct dm_buffer *b)
1497 {
1498         return b + 1;
1499 }
1500 EXPORT_SYMBOL_GPL(dm_bufio_get_aux_data);
1501
1502 struct dm_bufio_client *dm_bufio_get_client(struct dm_buffer *b)
1503 {
1504         return b->c;
1505 }
1506 EXPORT_SYMBOL_GPL(dm_bufio_get_client);
1507
1508 static void drop_buffers(struct dm_bufio_client *c)
1509 {
1510         struct dm_buffer *b;
1511         int i;
1512         bool warned = false;
1513
1514         BUG_ON(dm_bufio_in_request());
1515
1516         /*
1517          * An optimization so that the buffers are not written one-by-one.
1518          */
1519         dm_bufio_write_dirty_buffers_async(c);
1520
1521         dm_bufio_lock(c);
1522
1523         while ((b = __get_unclaimed_buffer(c)))
1524                 __free_buffer_wake(b);
1525
1526         for (i = 0; i < LIST_SIZE; i++)
1527                 list_for_each_entry(b, &c->lru[i], lru_list) {
1528                         WARN_ON(!warned);
1529                         warned = true;
1530                         DMERR("leaked buffer %llx, hold count %u, list %d",
1531                               (unsigned long long)b->block, b->hold_count, i);
1532 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
1533                         print_stack_trace(&b->stack_trace, 1);
1534                         b->hold_count = 0; /* mark unclaimed to avoid BUG_ON below */
1535 #endif
1536                 }
1537
1538 #ifdef CONFIG_DM_DEBUG_BLOCK_STACK_TRACING
1539         while ((b = __get_unclaimed_buffer(c)))
1540                 __free_buffer_wake(b);
1541 #endif
1542
1543         for (i = 0; i < LIST_SIZE; i++)
1544                 BUG_ON(!list_empty(&c->lru[i]));
1545
1546         dm_bufio_unlock(c);
1547 }
1548
1549 /*
1550  * We may not be able to evict this buffer if IO pending or the client
1551  * is still using it.  Caller is expected to know buffer is too old.
1552  *
1553  * And if GFP_NOFS is used, we must not do any I/O because we hold
1554  * dm_bufio_clients_lock and we would risk deadlock if the I/O gets
1555  * rerouted to different bufio client.
1556  */
1557 static bool __try_evict_buffer(struct dm_buffer *b, gfp_t gfp)
1558 {
1559         if (!(gfp & __GFP_FS)) {
1560                 if (test_bit(B_READING, &b->state) ||
1561                     test_bit(B_WRITING, &b->state) ||
1562                     test_bit(B_DIRTY, &b->state))
1563                         return false;
1564         }
1565
1566         if (b->hold_count)
1567                 return false;
1568
1569         __make_buffer_clean(b);
1570         __unlink_buffer(b);
1571         __free_buffer_wake(b);
1572
1573         return true;
1574 }
1575
1576 static unsigned long get_retain_buffers(struct dm_bufio_client *c)
1577 {
1578         unsigned long retain_bytes = READ_ONCE(dm_bufio_retain_bytes);
1579         if (likely(c->sectors_per_block_bits >= 0))
1580                 retain_bytes >>= c->sectors_per_block_bits + SECTOR_SHIFT;
1581         else
1582                 retain_bytes /= c->block_size;
1583         return retain_bytes;
1584 }
1585
1586 static unsigned long __scan(struct dm_bufio_client *c, unsigned long nr_to_scan,
1587                             gfp_t gfp_mask)
1588 {
1589         int l;
1590         struct dm_buffer *b, *tmp;
1591         unsigned long freed = 0;
1592         unsigned long count = c->n_buffers[LIST_CLEAN] +
1593                               c->n_buffers[LIST_DIRTY];
1594         unsigned long retain_target = get_retain_buffers(c);
1595
1596         for (l = 0; l < LIST_SIZE; l++) {
1597                 list_for_each_entry_safe_reverse(b, tmp, &c->lru[l], lru_list) {
1598                         if (__try_evict_buffer(b, gfp_mask))
1599                                 freed++;
1600                         if (!--nr_to_scan || ((count - freed) <= retain_target))
1601                                 return freed;
1602                         cond_resched();
1603                 }
1604         }
1605         return freed;
1606 }
1607
1608 static unsigned long
1609 dm_bufio_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
1610 {
1611         struct dm_bufio_client *c;
1612         unsigned long freed;
1613
1614         c = container_of(shrink, struct dm_bufio_client, shrinker);
1615         if (sc->gfp_mask & __GFP_FS)
1616                 dm_bufio_lock(c);
1617         else if (!dm_bufio_trylock(c))
1618                 return SHRINK_STOP;
1619
1620         freed  = __scan(c, sc->nr_to_scan, sc->gfp_mask);
1621         dm_bufio_unlock(c);
1622         return freed;
1623 }
1624
1625 static unsigned long
1626 dm_bufio_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
1627 {
1628         struct dm_bufio_client *c = container_of(shrink, struct dm_bufio_client, shrinker);
1629         unsigned long count = READ_ONCE(c->n_buffers[LIST_CLEAN]) +
1630                               READ_ONCE(c->n_buffers[LIST_DIRTY]);
1631         unsigned long retain_target = get_retain_buffers(c);
1632
1633         return (count < retain_target) ? 0 : (count - retain_target);
1634 }
1635
1636 /*
1637  * Create the buffering interface
1638  */
1639 struct dm_bufio_client *dm_bufio_client_create(struct block_device *bdev, unsigned block_size,
1640                                                unsigned reserved_buffers, unsigned aux_size,
1641                                                void (*alloc_callback)(struct dm_buffer *),
1642                                                void (*write_callback)(struct dm_buffer *))
1643 {
1644         int r;
1645         struct dm_bufio_client *c;
1646         unsigned i;
1647         char slab_name[27];
1648
1649         if (!block_size || block_size & ((1 << SECTOR_SHIFT) - 1)) {
1650                 DMERR("%s: block size not specified or is not multiple of 512b", __func__);
1651                 r = -EINVAL;
1652                 goto bad_client;
1653         }
1654
1655         c = kzalloc(sizeof(*c), GFP_KERNEL);
1656         if (!c) {
1657                 r = -ENOMEM;
1658                 goto bad_client;
1659         }
1660         c->buffer_tree = RB_ROOT;
1661
1662         c->bdev = bdev;
1663         c->block_size = block_size;
1664         if (is_power_of_2(block_size))
1665                 c->sectors_per_block_bits = __ffs(block_size) - SECTOR_SHIFT;
1666         else
1667                 c->sectors_per_block_bits = -1;
1668
1669         c->alloc_callback = alloc_callback;
1670         c->write_callback = write_callback;
1671
1672         for (i = 0; i < LIST_SIZE; i++) {
1673                 INIT_LIST_HEAD(&c->lru[i]);
1674                 c->n_buffers[i] = 0;
1675         }
1676
1677         mutex_init(&c->lock);
1678         INIT_LIST_HEAD(&c->reserved_buffers);
1679         c->need_reserved_buffers = reserved_buffers;
1680
1681         dm_bufio_set_minimum_buffers(c, DM_BUFIO_MIN_BUFFERS);
1682
1683         init_waitqueue_head(&c->free_buffer_wait);
1684         c->async_write_error = 0;
1685
1686         c->dm_io = dm_io_client_create();
1687         if (IS_ERR(c->dm_io)) {
1688                 r = PTR_ERR(c->dm_io);
1689                 goto bad_dm_io;
1690         }
1691
1692         if (block_size <= KMALLOC_MAX_SIZE &&
1693             (block_size < PAGE_SIZE || !is_power_of_2(block_size))) {
1694                 unsigned align = min(1U << __ffs(block_size), (unsigned)PAGE_SIZE);
1695                 snprintf(slab_name, sizeof slab_name, "dm_bufio_cache-%u", block_size);
1696                 c->slab_cache = kmem_cache_create(slab_name, block_size, align,
1697                                                   SLAB_RECLAIM_ACCOUNT, NULL);
1698                 if (!c->slab_cache) {
1699                         r = -ENOMEM;
1700                         goto bad;
1701                 }
1702         }
1703         if (aux_size)
1704                 snprintf(slab_name, sizeof slab_name, "dm_bufio_buffer-%u", aux_size);
1705         else
1706                 snprintf(slab_name, sizeof slab_name, "dm_bufio_buffer");
1707         c->slab_buffer = kmem_cache_create(slab_name, sizeof(struct dm_buffer) + aux_size,
1708                                            0, SLAB_RECLAIM_ACCOUNT, NULL);
1709         if (!c->slab_buffer) {
1710                 r = -ENOMEM;
1711                 goto bad;
1712         }
1713
1714         while (c->need_reserved_buffers) {
1715                 struct dm_buffer *b = alloc_buffer(c, GFP_KERNEL);
1716
1717                 if (!b) {
1718                         r = -ENOMEM;
1719                         goto bad;
1720                 }
1721                 __free_buffer_wake(b);
1722         }
1723
1724         c->shrinker.count_objects = dm_bufio_shrink_count;
1725         c->shrinker.scan_objects = dm_bufio_shrink_scan;
1726         c->shrinker.seeks = 1;
1727         c->shrinker.batch = 0;
1728         r = register_shrinker(&c->shrinker);
1729         if (r)
1730                 goto bad;
1731
1732         mutex_lock(&dm_bufio_clients_lock);
1733         dm_bufio_client_count++;
1734         list_add(&c->client_list, &dm_bufio_all_clients);
1735         __cache_size_refresh();
1736         mutex_unlock(&dm_bufio_clients_lock);
1737
1738         return c;
1739
1740 bad:
1741         while (!list_empty(&c->reserved_buffers)) {
1742                 struct dm_buffer *b = list_entry(c->reserved_buffers.next,
1743                                                  struct dm_buffer, lru_list);
1744                 list_del(&b->lru_list);
1745                 free_buffer(b);
1746         }
1747         kmem_cache_destroy(c->slab_cache);
1748         kmem_cache_destroy(c->slab_buffer);
1749         dm_io_client_destroy(c->dm_io);
1750 bad_dm_io:
1751         mutex_destroy(&c->lock);
1752         kfree(c);
1753 bad_client:
1754         return ERR_PTR(r);
1755 }
1756 EXPORT_SYMBOL_GPL(dm_bufio_client_create);
1757
1758 /*
1759  * Free the buffering interface.
1760  * It is required that there are no references on any buffers.
1761  */
1762 void dm_bufio_client_destroy(struct dm_bufio_client *c)
1763 {
1764         unsigned i;
1765
1766         drop_buffers(c);
1767
1768         unregister_shrinker(&c->shrinker);
1769
1770         mutex_lock(&dm_bufio_clients_lock);
1771
1772         list_del(&c->client_list);
1773         dm_bufio_client_count--;
1774         __cache_size_refresh();
1775
1776         mutex_unlock(&dm_bufio_clients_lock);
1777
1778         BUG_ON(!RB_EMPTY_ROOT(&c->buffer_tree));
1779         BUG_ON(c->need_reserved_buffers);
1780
1781         while (!list_empty(&c->reserved_buffers)) {
1782                 struct dm_buffer *b = list_entry(c->reserved_buffers.next,
1783                                                  struct dm_buffer, lru_list);
1784                 list_del(&b->lru_list);
1785                 free_buffer(b);
1786         }
1787
1788         for (i = 0; i < LIST_SIZE; i++)
1789                 if (c->n_buffers[i])
1790                         DMERR("leaked buffer count %d: %ld", i, c->n_buffers[i]);
1791
1792         for (i = 0; i < LIST_SIZE; i++)
1793                 BUG_ON(c->n_buffers[i]);
1794
1795         kmem_cache_destroy(c->slab_cache);
1796         kmem_cache_destroy(c->slab_buffer);
1797         dm_io_client_destroy(c->dm_io);
1798         mutex_destroy(&c->lock);
1799         kfree(c);
1800 }
1801 EXPORT_SYMBOL_GPL(dm_bufio_client_destroy);
1802
1803 void dm_bufio_set_sector_offset(struct dm_bufio_client *c, sector_t start)
1804 {
1805         c->start = start;
1806 }
1807 EXPORT_SYMBOL_GPL(dm_bufio_set_sector_offset);
1808
1809 static unsigned get_max_age_hz(void)
1810 {
1811         unsigned max_age = READ_ONCE(dm_bufio_max_age);
1812
1813         if (max_age > UINT_MAX / HZ)
1814                 max_age = UINT_MAX / HZ;
1815
1816         return max_age * HZ;
1817 }
1818
1819 static bool older_than(struct dm_buffer *b, unsigned long age_hz)
1820 {
1821         return time_after_eq(jiffies, b->last_accessed + age_hz);
1822 }
1823
1824 static void __evict_old_buffers(struct dm_bufio_client *c, unsigned long age_hz)
1825 {
1826         struct dm_buffer *b, *tmp;
1827         unsigned long retain_target = get_retain_buffers(c);
1828         unsigned long count;
1829         LIST_HEAD(write_list);
1830
1831         dm_bufio_lock(c);
1832
1833         __check_watermark(c, &write_list);
1834         if (unlikely(!list_empty(&write_list))) {
1835                 dm_bufio_unlock(c);
1836                 __flush_write_list(&write_list);
1837                 dm_bufio_lock(c);
1838         }
1839
1840         count = c->n_buffers[LIST_CLEAN] + c->n_buffers[LIST_DIRTY];
1841         list_for_each_entry_safe_reverse(b, tmp, &c->lru[LIST_CLEAN], lru_list) {
1842                 if (count <= retain_target)
1843                         break;
1844
1845                 if (!older_than(b, age_hz))
1846                         break;
1847
1848                 if (__try_evict_buffer(b, 0))
1849                         count--;
1850
1851                 cond_resched();
1852         }
1853
1854         dm_bufio_unlock(c);
1855 }
1856
1857 static void cleanup_old_buffers(void)
1858 {
1859         unsigned long max_age_hz = get_max_age_hz();
1860         struct dm_bufio_client *c;
1861
1862         mutex_lock(&dm_bufio_clients_lock);
1863
1864         __cache_size_refresh();
1865
1866         list_for_each_entry(c, &dm_bufio_all_clients, client_list)
1867                 __evict_old_buffers(c, max_age_hz);
1868
1869         mutex_unlock(&dm_bufio_clients_lock);
1870 }
1871
1872 static struct workqueue_struct *dm_bufio_wq;
1873 static struct delayed_work dm_bufio_work;
1874
1875 static void work_fn(struct work_struct *w)
1876 {
1877         cleanup_old_buffers();
1878
1879         queue_delayed_work(dm_bufio_wq, &dm_bufio_work,
1880                            DM_BUFIO_WORK_TIMER_SECS * HZ);
1881 }
1882
1883 /*----------------------------------------------------------------
1884  * Module setup
1885  *--------------------------------------------------------------*/
1886
1887 /*
1888  * This is called only once for the whole dm_bufio module.
1889  * It initializes memory limit.
1890  */
1891 static int __init dm_bufio_init(void)
1892 {
1893         __u64 mem;
1894
1895         dm_bufio_allocated_kmem_cache = 0;
1896         dm_bufio_allocated_get_free_pages = 0;
1897         dm_bufio_allocated_vmalloc = 0;
1898         dm_bufio_current_allocated = 0;
1899
1900         mem = (__u64)mult_frac(totalram_pages - totalhigh_pages,
1901                                DM_BUFIO_MEMORY_PERCENT, 100) << PAGE_SHIFT;
1902
1903         if (mem > ULONG_MAX)
1904                 mem = ULONG_MAX;
1905
1906 #ifdef CONFIG_MMU
1907         if (mem > mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100))
1908                 mem = mult_frac(VMALLOC_TOTAL, DM_BUFIO_VMALLOC_PERCENT, 100);
1909 #endif
1910
1911         dm_bufio_default_cache_size = mem;
1912
1913         mutex_lock(&dm_bufio_clients_lock);
1914         __cache_size_refresh();
1915         mutex_unlock(&dm_bufio_clients_lock);
1916
1917         dm_bufio_wq = alloc_workqueue("dm_bufio_cache", WQ_MEM_RECLAIM, 0);
1918         if (!dm_bufio_wq)
1919                 return -ENOMEM;
1920
1921         INIT_DELAYED_WORK(&dm_bufio_work, work_fn);
1922         queue_delayed_work(dm_bufio_wq, &dm_bufio_work,
1923                            DM_BUFIO_WORK_TIMER_SECS * HZ);
1924
1925         return 0;
1926 }
1927
1928 /*
1929  * This is called once when unloading the dm_bufio module.
1930  */
1931 static void __exit dm_bufio_exit(void)
1932 {
1933         int bug = 0;
1934
1935         cancel_delayed_work_sync(&dm_bufio_work);
1936         destroy_workqueue(dm_bufio_wq);
1937
1938         if (dm_bufio_client_count) {
1939                 DMCRIT("%s: dm_bufio_client_count leaked: %d",
1940                         __func__, dm_bufio_client_count);
1941                 bug = 1;
1942         }
1943
1944         if (dm_bufio_current_allocated) {
1945                 DMCRIT("%s: dm_bufio_current_allocated leaked: %lu",
1946                         __func__, dm_bufio_current_allocated);
1947                 bug = 1;
1948         }
1949
1950         if (dm_bufio_allocated_get_free_pages) {
1951                 DMCRIT("%s: dm_bufio_allocated_get_free_pages leaked: %lu",
1952                        __func__, dm_bufio_allocated_get_free_pages);
1953                 bug = 1;
1954         }
1955
1956         if (dm_bufio_allocated_vmalloc) {
1957                 DMCRIT("%s: dm_bufio_vmalloc leaked: %lu",
1958                        __func__, dm_bufio_allocated_vmalloc);
1959                 bug = 1;
1960         }
1961
1962         BUG_ON(bug);
1963 }
1964
1965 module_init(dm_bufio_init)
1966 module_exit(dm_bufio_exit)
1967
1968 module_param_named(max_cache_size_bytes, dm_bufio_cache_size, ulong, S_IRUGO | S_IWUSR);
1969 MODULE_PARM_DESC(max_cache_size_bytes, "Size of metadata cache");
1970
1971 module_param_named(max_age_seconds, dm_bufio_max_age, uint, S_IRUGO | S_IWUSR);
1972 MODULE_PARM_DESC(max_age_seconds, "Max age of a buffer in seconds");
1973
1974 module_param_named(retain_bytes, dm_bufio_retain_bytes, ulong, S_IRUGO | S_IWUSR);
1975 MODULE_PARM_DESC(retain_bytes, "Try to keep at least this many bytes cached in memory");
1976
1977 module_param_named(peak_allocated_bytes, dm_bufio_peak_allocated, ulong, S_IRUGO | S_IWUSR);
1978 MODULE_PARM_DESC(peak_allocated_bytes, "Tracks the maximum allocated memory");
1979
1980 module_param_named(allocated_kmem_cache_bytes, dm_bufio_allocated_kmem_cache, ulong, S_IRUGO);
1981 MODULE_PARM_DESC(allocated_kmem_cache_bytes, "Memory allocated with kmem_cache_alloc");
1982
1983 module_param_named(allocated_get_free_pages_bytes, dm_bufio_allocated_get_free_pages, ulong, S_IRUGO);
1984 MODULE_PARM_DESC(allocated_get_free_pages_bytes, "Memory allocated with get_free_pages");
1985
1986 module_param_named(allocated_vmalloc_bytes, dm_bufio_allocated_vmalloc, ulong, S_IRUGO);
1987 MODULE_PARM_DESC(allocated_vmalloc_bytes, "Memory allocated with vmalloc");
1988
1989 module_param_named(current_allocated_bytes, dm_bufio_current_allocated, ulong, S_IRUGO);
1990 MODULE_PARM_DESC(current_allocated_bytes, "Memory currently used by the cache");
1991
1992 MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");
1993 MODULE_DESCRIPTION(DM_NAME " buffered I/O library");
1994 MODULE_LICENSE("GPL");