2 * Copyright (c) 1998 Michael Smith <msmith@freebsd.org>
3 * Copyright 2015 Toomas Soome <tsoome@me.com>
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 #include <sys/cdefs.h>
29 #include <sys/param.h>
30 __FBSDID("$FreeBSD$");
33 * Simple hashed block cache
36 #include <sys/stdint.h>
42 #include "bootstrap.h"
44 /* #define BCACHE_DEBUG */
47 # define DEBUG(fmt, args...) printf("%s: " fmt "\n" , __func__ , ## args)
49 # define DEBUG(fmt, args...)
59 * bcache per device node. cache is allocated on device first open and freed
60 * on last close, to save memory. The issue there is the size; biosdisk
61 * supports up to 31 (0x1f) devices. Classic setup would use single disk
62 * to boot from, but this has changed with zfs.
65 struct bcachectl *bcache_ctl;
71 static u_int bcache_total_nblks; /* set by bcache_init */
72 static u_int bcache_blksize; /* set by bcache_init */
73 static u_int bcache_numdev; /* set by bcache_add_dev */
75 static u_int bcache_units; /* number of devices with cache */
76 static u_int bcache_unit_nblks; /* nblocks per unit */
77 static u_int bcache_hits;
78 static u_int bcache_misses;
79 static u_int bcache_ops;
80 static u_int bcache_bypasses;
81 static u_int bcache_bcount;
82 static u_int bcache_rablks;
84 #define BHASH(bc, blkno) ((blkno) & ((bc)->bcache_nblks - 1))
85 #define BCACHE_LOOKUP(bc, blkno) \
86 ((bc)->bcache_ctl[BHASH((bc), (blkno))].bc_blkno != (blkno))
87 #define BCACHE_READAHEAD 256
88 #define BCACHE_MINREADAHEAD 32
89 #define BCACHE_MARKER 0xdeadbeef
91 static void bcache_invalidate(struct bcache *bc, daddr_t blkno);
92 static void bcache_insert(struct bcache *bc, daddr_t blkno);
93 static void bcache_free_instance(struct bcache *bc);
96 * Initialise the cache for (nblks) of (bsize).
99 bcache_init(size_t nblks, size_t bsize)
101 /* set up control data */
102 bcache_total_nblks = nblks;
103 bcache_blksize = bsize;
107 * add number of devices to bcache. we have to divide cache space
108 * between the devices, so bcache_add_dev() can be used to set up the
109 * number. The issue is, we need to get the number before actual allocations.
110 * bcache_add_dev() is supposed to be called from device init() call, so the
111 * assumption is, devsw dv_init is called for plain devices first, and
115 bcache_add_dev(int devices)
117 bcache_numdev += devices;
121 bcache_allocate(void)
124 struct bcache *bc = malloc(sizeof (struct bcache));
125 int disks = bcache_numdev;
129 disks = 1; /* safe guard */
137 * the bcache block count must be power of 2 for hash function
139 i = fls(disks) - 1; /* highbit - 1 */
140 if (disks > (1 << i)) /* next power of 2 */
143 bc->bcache_nblks = bcache_total_nblks >> i;
144 bcache_unit_nblks = bc->bcache_nblks;
145 bc->bcache_data = malloc(bc->bcache_nblks * bcache_blksize +
147 if (bc->bcache_data == NULL) {
148 /* dont error out yet. fall back to 32 blocks and try again */
149 bc->bcache_nblks = 32;
150 bc->bcache_data = malloc(bc->bcache_nblks * bcache_blksize +
154 bc->bcache_ctl = malloc(bc->bcache_nblks * sizeof(struct bcachectl));
156 if ((bc->bcache_data == NULL) || (bc->bcache_ctl == NULL)) {
157 bcache_free_instance(bc);
161 /* Insert cache end marker. */
162 marker = (uint32_t *)(bc->bcache_data + bc->bcache_nblks * bcache_blksize);
163 *marker = BCACHE_MARKER;
165 /* Flush the cache */
166 for (i = 0; i < bc->bcache_nblks; i++) {
167 bc->bcache_ctl[i].bc_count = -1;
168 bc->bcache_ctl[i].bc_blkno = -1;
171 bc->ra = BCACHE_READAHEAD; /* optimistic read ahead */
176 bcache_free(void *cache)
178 struct bcache *bc = cache;
183 bcache_free_instance(bc);
188 * Handle a write request; write directly to the disk, and populate the
189 * cache with the new values.
192 write_strategy(void *devdata, int rw, daddr_t blk, size_t size,
193 char *buf, size_t *rsize)
195 struct bcache_devdata *dd = (struct bcache_devdata *)devdata;
196 struct bcache *bc = dd->dv_cache;
199 nblk = size / bcache_blksize;
201 /* Invalidate the blocks being written */
202 for (i = 0; i < nblk; i++) {
203 bcache_invalidate(bc, blk + i);
206 /* Write the blocks */
207 return (dd->dv_strategy(dd->dv_devdata, rw, blk, size, buf, rsize));
211 * Handle a read request; fill in parts of the request that can
212 * be satisfied by the cache, use the supplied strategy routine to do
213 * device I/O and then use the I/O results to populate the cache.
216 read_strategy(void *devdata, int rw, daddr_t blk, size_t size,
217 char *buf, size_t *rsize)
219 struct bcache_devdata *dd = (struct bcache_devdata *)devdata;
220 struct bcache *bc = dd->dv_cache;
221 size_t i, nblk, p_size, r_size, complete, ra;
232 marker = (uint32_t *)(bc->bcache_data + bc->bcache_nblks * bcache_blksize);
237 nblk = size / bcache_blksize;
238 if (nblk == 0 && size != 0)
243 /* Satisfy any cache hits up front, break on first miss */
244 for (i = 0; i < nblk; i++) {
245 if (BCACHE_LOOKUP(bc, (daddr_t)(blk + i))) {
246 bcache_misses += (nblk - i);
248 if (nblk - i > BCACHE_MINREADAHEAD && bc->ra > BCACHE_MINREADAHEAD)
249 bc->ra >>= 1; /* reduce read ahead */
256 if (complete) { /* whole set was in cache, return it */
257 if (bc->ra < BCACHE_READAHEAD)
258 bc->ra <<= 1; /* increase read ahead */
259 bcopy(bc->bcache_data + (bcache_blksize * BHASH(bc, blk)), buf, size);
264 * Fill in any misses. From check we have i pointing to first missing
265 * block, read in all remaining blocks + readahead.
266 * We have space at least for nblk - i before bcache wraps.
269 p_buf = bc->bcache_data + (bcache_blksize * BHASH(bc, p_blk));
270 r_size = bc->bcache_nblks - BHASH(bc, p_blk); /* remaining blocks */
272 p_size = MIN(r_size, nblk - i); /* read at least those blocks */
275 * The read ahead size setup.
276 * While the read ahead can save us IO, it also can complicate things:
277 * 1. We do not want to read ahead by wrapping around the
278 * bcache end - this would complicate the cache management.
279 * 2. We are using bc->ra as dynamic hint for read ahead size,
280 * detected cache hits will increase the read-ahead block count, and
281 * misses will decrease, see the code above.
282 * 3. The bcache is sized by 512B blocks, however, the underlying device
283 * may have a larger sector size, and we should perform the IO by
284 * taking into account these larger sector sizes. We could solve this by
285 * passing the sector size to bcache_allocate(), or by using ioctl(), but
286 * in this version we are using the constant, 16 blocks, and are rounding
287 * read ahead block count down to multiple of 16.
288 * Using the constant has two reasons, we are not entirely sure if the
289 * BIOS disk interface is providing the correct value for sector size.
290 * And secondly, this way we get the most conservative setup for the ra.
292 * The selection of multiple of 16 blocks (8KB) is quite arbitrary, however,
293 * we want to cover CDs (2K) and 4K disks.
294 * bcache_allocate() will always fall back to a minimum of 32 blocks.
295 * Our choice of 16 read ahead blocks will always fit inside the bcache.
298 ra = bc->bcache_nblks - BHASH(bc, p_blk + p_size);
299 if (ra != 0 && ra != bc->bcache_nblks) { /* do we have RA space? */
300 ra = MIN(bc->ra, ra - 1);
301 ra = rounddown(ra, 16); /* multiple of 16 blocks */
305 /* invalidate bcache */
306 for (i = 0; i < p_size; i++) {
307 bcache_invalidate(bc, p_blk + i);
312 * with read-ahead, it may happen we are attempting to read past
313 * disk end, as bcache has no information about disk size.
314 * in such case we should get partial read if some blocks can be
315 * read or error, if no blocks can be read.
316 * in either case we should return the data in bcache and only
317 * return error if there is no data.
319 result = dd->dv_strategy(dd->dv_devdata, rw, p_blk,
320 p_size * bcache_blksize, p_buf, &r_size);
322 r_size /= bcache_blksize;
323 for (i = 0; i < r_size; i++)
324 bcache_insert(bc, p_blk + i);
326 /* update ra statistics */
329 bcache_rablks += (p_size - r_size);
334 /* check how much data can we copy */
335 for (i = 0; i < nblk; i++) {
336 if (BCACHE_LOOKUP(bc, (daddr_t)(blk + i)))
340 if (size > i * bcache_blksize)
341 size = i * bcache_blksize;
344 bcopy(bc->bcache_data + (bcache_blksize * BHASH(bc, blk)), buf, size);
348 if (*marker != BCACHE_MARKER) {
349 printf("BUG: bcache corruption detected: nblks: %zu p_blk: %lu, "
350 "p_size: %zu, ra: %zu\n", bc->bcache_nblks,
351 (long unsigned)BHASH(bc, p_blk), p_size, ra);
355 if ((result == 0) && (rsize != NULL))
361 * Requests larger than 1/2 cache size will be bypassed and go
362 * directly to the disk. XXX tune this.
365 bcache_strategy(void *devdata, int rw, daddr_t blk, size_t size,
366 char *buf, size_t *rsize)
368 struct bcache_devdata *dd = (struct bcache_devdata *)devdata;
369 struct bcache *bc = dd->dv_cache;
370 u_int bcache_nblks = 0;
372 size_t csize, isize, total;
377 bcache_nblks = bc->bcache_nblks;
379 /* bypass large requests, or when the cache is inactive */
381 ((size * 2 / bcache_blksize) > bcache_nblks)) {
382 DEBUG("bypass %zu from %qu", size / bcache_blksize, blk);
384 return (dd->dv_strategy(dd->dv_devdata, rw, blk, size, buf, rsize));
389 nblk = size / bcache_blksize;
390 if (size != 0 && nblk == 0)
391 nblk++; /* read at least one block */
396 cblk = bcache_nblks - BHASH(bc, blk); /* # of blocks left */
397 cblk = MIN(cblk, nblk);
399 if (size <= bcache_blksize)
402 csize = cblk * bcache_blksize;
404 ret = read_strategy(devdata, rw, blk, csize, buf+total, &isize);
407 * we may have error from read ahead, if we have read some data
408 * return partial read.
410 if (ret != 0 || isize == 0) {
415 blk += isize / bcache_blksize;
418 nblk = size / bcache_blksize;
426 return write_strategy(devdata, rw, blk, size, buf, rsize);
432 * Free allocated bcache instance
435 bcache_free_instance(struct bcache *bc)
439 free(bc->bcache_ctl);
441 free(bc->bcache_data);
447 * Insert a block into the cache.
450 bcache_insert(struct bcache *bc, daddr_t blkno)
454 cand = BHASH(bc, blkno);
456 DEBUG("insert blk %llu -> %u # %d", blkno, cand, bcache_bcount);
457 bc->bcache_ctl[cand].bc_blkno = blkno;
458 bc->bcache_ctl[cand].bc_count = bcache_bcount++;
462 * Invalidate a block from the cache.
465 bcache_invalidate(struct bcache *bc, daddr_t blkno)
469 i = BHASH(bc, blkno);
470 if (bc->bcache_ctl[i].bc_blkno == blkno) {
471 bc->bcache_ctl[i].bc_count = -1;
472 bc->bcache_ctl[i].bc_blkno = -1;
473 DEBUG("invalidate blk %llu", blkno);
478 COMMAND_SET(bcachestat, "bcachestat", "get disk block cache stats", command_bcache);
481 command_bcache(int argc, char *argv[])
484 command_errmsg = "wrong number of arguments";
488 printf("\ncache blocks: %d\n", bcache_total_nblks);
489 printf("cache blocksz: %d\n", bcache_blksize);
490 printf("cache readahead: %d\n", bcache_rablks);
491 printf("unit cache blocks: %d\n", bcache_unit_nblks);
492 printf("cached units: %d\n", bcache_units);
493 printf("%d ops %d bypasses %d hits %d misses\n", bcache_ops,
494 bcache_bypasses, bcache_hits, bcache_misses);