4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
25 #include <sys/dataset_kstats.h>
27 #include <sys/dmu_traverse.h>
28 #include <sys/dsl_dataset.h>
29 #include <sys/dsl_prop.h>
30 #include <sys/dsl_dir.h>
32 #include <sys/zfeature.h>
33 #include <sys/zil_impl.h>
34 #include <sys/dmu_tx.h>
36 #include <sys/zfs_rlock.h>
37 #include <sys/spa_impl.h>
39 #include <sys/zvol_impl.h>
41 #include <linux/blkdev_compat.h>
42 #include <linux/task_io_accounting_ops.h>
44 static unsigned int zvol_major = ZVOL_MAJOR;
45 static unsigned int zvol_request_sync = 0;
46 static unsigned int zvol_prefetch_bytes = (128 * 1024);
47 static unsigned long zvol_max_discard_blocks = 16384;
48 static unsigned int zvol_threads = 32;
49 static const unsigned int zvol_open_timeout_ms = 1000;
51 struct zvol_state_os {
52 struct gendisk *zvo_disk; /* generic disk */
53 struct request_queue *zvo_queue; /* request queue */
54 dev_t zvo_dev; /* device id */
58 static struct ida zvol_ida;
60 typedef struct zv_request_stack {
65 typedef struct zv_request_task {
70 static zv_request_task_t *
71 zv_request_task_create(zv_request_t zvr)
73 zv_request_task_t *task;
74 task = kmem_alloc(sizeof (zv_request_task_t), KM_SLEEP);
75 taskq_init_ent(&task->ent);
81 zv_request_task_free(zv_request_task_t *task)
83 kmem_free(task, sizeof (*task));
87 * Given a path, return TRUE if path is a ZVOL.
90 zvol_os_is_zvol(const char *path)
94 if (vdev_lookup_bdev(path, &dev) != 0)
97 if (MAJOR(dev) == zvol_major)
104 zvol_write(zv_request_t *zvr)
106 struct bio *bio = zvr->bio;
110 zfs_uio_bvec_init(&uio, bio);
112 zvol_state_t *zv = zvr->zv;
113 ASSERT3P(zv, !=, NULL);
114 ASSERT3U(zv->zv_open_count, >, 0);
115 ASSERT3P(zv->zv_zilog, !=, NULL);
117 /* bio marked as FLUSH need to flush before write */
118 if (bio_is_flush(bio))
119 zil_commit(zv->zv_zilog, ZVOL_OBJ);
121 /* Some requests are just for flush and nothing else. */
122 if (uio.uio_resid == 0) {
123 rw_exit(&zv->zv_suspend_lock);
128 struct request_queue *q = zv->zv_zso->zvo_queue;
129 struct gendisk *disk = zv->zv_zso->zvo_disk;
130 ssize_t start_resid = uio.uio_resid;
131 unsigned long start_time;
133 boolean_t acct = blk_queue_io_stat(q);
135 start_time = blk_generic_start_io_acct(q, disk, WRITE, bio);
138 bio_is_fua(bio) || zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS;
140 zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
141 uio.uio_loffset, uio.uio_resid, RL_WRITER);
143 uint64_t volsize = zv->zv_volsize;
144 while (uio.uio_resid > 0 && uio.uio_loffset < volsize) {
145 uint64_t bytes = MIN(uio.uio_resid, DMU_MAX_ACCESS >> 1);
146 uint64_t off = uio.uio_loffset;
147 dmu_tx_t *tx = dmu_tx_create(zv->zv_objset);
149 if (bytes > volsize - off) /* don't write past the end */
150 bytes = volsize - off;
152 dmu_tx_hold_write_by_dnode(tx, zv->zv_dn, off, bytes);
154 /* This will only fail for ENOSPC */
155 error = dmu_tx_assign(tx, TXG_WAIT);
160 error = dmu_write_uio_dnode(zv->zv_dn, &uio, bytes, tx);
162 zvol_log_write(zv, tx, off, bytes, sync);
169 zfs_rangelock_exit(lr);
171 int64_t nwritten = start_resid - uio.uio_resid;
172 dataset_kstats_update_write_kstats(&zv->zv_kstat, nwritten);
173 task_io_account_write(nwritten);
176 zil_commit(zv->zv_zilog, ZVOL_OBJ);
178 rw_exit(&zv->zv_suspend_lock);
181 blk_generic_end_io_acct(q, disk, WRITE, bio, start_time);
183 BIO_END_IO(bio, -error);
187 zvol_write_task(void *arg)
189 zv_request_task_t *task = arg;
190 zvol_write(&task->zvr);
191 zv_request_task_free(task);
195 zvol_discard(zv_request_t *zvr)
197 struct bio *bio = zvr->bio;
198 zvol_state_t *zv = zvr->zv;
199 uint64_t start = BIO_BI_SECTOR(bio) << 9;
200 uint64_t size = BIO_BI_SIZE(bio);
201 uint64_t end = start + size;
206 ASSERT3P(zv, !=, NULL);
207 ASSERT3U(zv->zv_open_count, >, 0);
208 ASSERT3P(zv->zv_zilog, !=, NULL);
210 struct request_queue *q = zv->zv_zso->zvo_queue;
211 struct gendisk *disk = zv->zv_zso->zvo_disk;
212 unsigned long start_time;
214 boolean_t acct = blk_queue_io_stat(q);
216 start_time = blk_generic_start_io_acct(q, disk, WRITE, bio);
218 sync = bio_is_fua(bio) || zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS;
220 if (end > zv->zv_volsize) {
221 error = SET_ERROR(EIO);
226 * Align the request to volume block boundaries when a secure erase is
227 * not required. This will prevent dnode_free_range() from zeroing out
228 * the unaligned parts which is slow (read-modify-write) and useless
229 * since we are not freeing any space by doing so.
231 if (!bio_is_secure_erase(bio)) {
232 start = P2ROUNDUP(start, zv->zv_volblocksize);
233 end = P2ALIGN(end, zv->zv_volblocksize);
240 zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
241 start, size, RL_WRITER);
243 tx = dmu_tx_create(zv->zv_objset);
244 dmu_tx_mark_netfree(tx);
245 error = dmu_tx_assign(tx, TXG_WAIT);
249 zvol_log_truncate(zv, tx, start, size, B_TRUE);
251 error = dmu_free_long_range(zv->zv_objset,
252 ZVOL_OBJ, start, size);
254 zfs_rangelock_exit(lr);
256 if (error == 0 && sync)
257 zil_commit(zv->zv_zilog, ZVOL_OBJ);
260 rw_exit(&zv->zv_suspend_lock);
263 blk_generic_end_io_acct(q, disk, WRITE, bio, start_time);
265 BIO_END_IO(bio, -error);
269 zvol_discard_task(void *arg)
271 zv_request_task_t *task = arg;
272 zvol_discard(&task->zvr);
273 zv_request_task_free(task);
277 zvol_read(zv_request_t *zvr)
279 struct bio *bio = zvr->bio;
283 zfs_uio_bvec_init(&uio, bio);
285 zvol_state_t *zv = zvr->zv;
286 ASSERT3P(zv, !=, NULL);
287 ASSERT3U(zv->zv_open_count, >, 0);
289 struct request_queue *q = zv->zv_zso->zvo_queue;
290 struct gendisk *disk = zv->zv_zso->zvo_disk;
291 ssize_t start_resid = uio.uio_resid;
292 unsigned long start_time;
294 boolean_t acct = blk_queue_io_stat(q);
296 start_time = blk_generic_start_io_acct(q, disk, READ, bio);
298 zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
299 uio.uio_loffset, uio.uio_resid, RL_READER);
301 uint64_t volsize = zv->zv_volsize;
302 while (uio.uio_resid > 0 && uio.uio_loffset < volsize) {
303 uint64_t bytes = MIN(uio.uio_resid, DMU_MAX_ACCESS >> 1);
305 /* don't read past the end */
306 if (bytes > volsize - uio.uio_loffset)
307 bytes = volsize - uio.uio_loffset;
309 error = dmu_read_uio_dnode(zv->zv_dn, &uio, bytes);
311 /* convert checksum errors into IO errors */
313 error = SET_ERROR(EIO);
317 zfs_rangelock_exit(lr);
319 int64_t nread = start_resid - uio.uio_resid;
320 dataset_kstats_update_read_kstats(&zv->zv_kstat, nread);
321 task_io_account_read(nread);
323 rw_exit(&zv->zv_suspend_lock);
326 blk_generic_end_io_acct(q, disk, READ, bio, start_time);
328 BIO_END_IO(bio, -error);
332 zvol_read_task(void *arg)
334 zv_request_task_t *task = arg;
335 zvol_read(&task->zvr);
336 zv_request_task_free(task);
339 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
340 #ifdef HAVE_BDEV_SUBMIT_BIO_RETURNS_VOID
342 zvol_submit_bio(struct bio *bio)
345 zvol_submit_bio(struct bio *bio)
348 static MAKE_REQUEST_FN_RET
349 zvol_request(struct request_queue *q, struct bio *bio)
352 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
353 #if defined(HAVE_BIO_BDEV_DISK)
354 struct request_queue *q = bio->bi_bdev->bd_disk->queue;
356 struct request_queue *q = bio->bi_disk->queue;
359 zvol_state_t *zv = q->queuedata;
360 fstrans_cookie_t cookie = spl_fstrans_mark();
361 uint64_t offset = BIO_BI_SECTOR(bio) << 9;
362 uint64_t size = BIO_BI_SIZE(bio);
363 int rw = bio_data_dir(bio);
365 if (bio_has_data(bio) && offset + size > zv->zv_volsize) {
367 "%s: bad access: offset=%llu, size=%lu\n",
368 zv->zv_zso->zvo_disk->disk_name,
369 (long long unsigned)offset,
370 (long unsigned)size);
372 BIO_END_IO(bio, -SET_ERROR(EIO));
380 zv_request_task_t *task;
383 if (unlikely(zv->zv_flags & ZVOL_RDONLY)) {
384 BIO_END_IO(bio, -SET_ERROR(EROFS));
389 * Prevents the zvol from being suspended, or the ZIL being
390 * concurrently opened. Will be released after the i/o
393 rw_enter(&zv->zv_suspend_lock, RW_READER);
396 * Open a ZIL if this is the first time we have written to this
397 * zvol. We protect zv->zv_zilog with zv_suspend_lock rather
398 * than zv_state_lock so that we don't need to acquire an
399 * additional lock in this path.
401 if (zv->zv_zilog == NULL) {
402 rw_exit(&zv->zv_suspend_lock);
403 rw_enter(&zv->zv_suspend_lock, RW_WRITER);
404 if (zv->zv_zilog == NULL) {
405 zv->zv_zilog = zil_open(zv->zv_objset,
407 zv->zv_flags |= ZVOL_WRITTEN_TO;
408 /* replay / destroy done in zvol_create_minor */
409 VERIFY0((zv->zv_zilog->zl_header->zh_flags &
412 rw_downgrade(&zv->zv_suspend_lock);
416 * We don't want this thread to be blocked waiting for i/o to
417 * complete, so we instead wait from a taskq callback. The
418 * i/o may be a ZIL write (via zil_commit()), or a read of an
419 * indirect block, or a read of a data block (if this is a
420 * partial-block write). We will indicate that the i/o is
421 * complete by calling BIO_END_IO() from the taskq callback.
423 * This design allows the calling thread to continue and
424 * initiate more concurrent operations by calling
425 * zvol_request() again. There are typically only a small
426 * number of threads available to call zvol_request() (e.g.
427 * one per iSCSI target), so keeping the latency of
428 * zvol_request() low is important for performance.
430 * The zvol_request_sync module parameter allows this
431 * behavior to be altered, for performance evaluation
432 * purposes. If the callback blocks, setting
433 * zvol_request_sync=1 will result in much worse performance.
435 * We can have up to zvol_threads concurrent i/o's being
436 * processed for all zvols on the system. This is typically
437 * a vast improvement over the zvol_request_sync=1 behavior
438 * of one i/o at a time per zvol. However, an even better
439 * design would be for zvol_request() to initiate the zio
440 * directly, and then be notified by the zio_done callback,
441 * which would call BIO_END_IO(). Unfortunately, the DMU/ZIL
442 * interfaces lack this functionality (they block waiting for
443 * the i/o to complete).
445 if (bio_is_discard(bio) || bio_is_secure_erase(bio)) {
446 if (zvol_request_sync) {
449 task = zv_request_task_create(zvr);
450 taskq_dispatch_ent(zvol_taskq,
451 zvol_discard_task, task, 0, &task->ent);
454 if (zvol_request_sync) {
457 task = zv_request_task_create(zvr);
458 taskq_dispatch_ent(zvol_taskq,
459 zvol_write_task, task, 0, &task->ent);
464 * The SCST driver, and possibly others, may issue READ I/Os
465 * with a length of zero bytes. These empty I/Os contain no
466 * data and require no additional handling.
473 rw_enter(&zv->zv_suspend_lock, RW_READER);
475 /* See comment in WRITE case above. */
476 if (zvol_request_sync) {
479 task = zv_request_task_create(zvr);
480 taskq_dispatch_ent(zvol_taskq,
481 zvol_read_task, task, 0, &task->ent);
486 spl_fstrans_unmark(cookie);
487 #if (defined(HAVE_MAKE_REQUEST_FN_RET_QC) || \
488 defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS)) && \
489 !defined(HAVE_BDEV_SUBMIT_BIO_RETURNS_VOID)
490 return (BLK_QC_T_NONE);
495 zvol_open(struct block_device *bdev, fmode_t flag)
499 boolean_t drop_suspend = B_FALSE;
500 #ifndef HAVE_BLKDEV_GET_ERESTARTSYS
501 hrtime_t timeout = MSEC2NSEC(zvol_open_timeout_ms);
502 hrtime_t start = gethrtime();
506 rw_enter(&zvol_state_lock, RW_READER);
508 * Obtain a copy of private_data under the zvol_state_lock to make
509 * sure that either the result of zvol free code path setting
510 * bdev->bd_disk->private_data to NULL is observed, or zvol_os_free()
511 * is not called on this zv because of the positive zv_open_count.
513 zv = bdev->bd_disk->private_data;
515 rw_exit(&zvol_state_lock);
516 return (SET_ERROR(-ENXIO));
519 mutex_enter(&zv->zv_state_lock);
521 * Make sure zvol is not suspended during first open
522 * (hold zv_suspend_lock) and respect proper lock acquisition
523 * ordering - zv_suspend_lock before zv_state_lock
525 if (zv->zv_open_count == 0) {
526 if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) {
527 mutex_exit(&zv->zv_state_lock);
528 rw_enter(&zv->zv_suspend_lock, RW_READER);
529 mutex_enter(&zv->zv_state_lock);
530 /* check to see if zv_suspend_lock is needed */
531 if (zv->zv_open_count != 0) {
532 rw_exit(&zv->zv_suspend_lock);
534 drop_suspend = B_TRUE;
537 drop_suspend = B_TRUE;
540 rw_exit(&zvol_state_lock);
542 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
544 if (zv->zv_open_count == 0) {
545 boolean_t drop_namespace = B_FALSE;
547 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
550 * In all other call paths the spa_namespace_lock is taken
551 * before the bdev->bd_mutex lock. However, on open(2)
552 * the __blkdev_get() function calls fops->open() with the
553 * bdev->bd_mutex lock held. This can result in a deadlock
554 * when zvols from one pool are used as vdevs in another.
556 * To prevent a lock inversion deadlock we preemptively
557 * take the spa_namespace_lock. Normally the lock will not
558 * be contended and this is safe because spa_open_common()
559 * handles the case where the caller already holds the
560 * spa_namespace_lock.
562 * When the lock cannot be aquired after multiple retries
563 * this must be the vdev on zvol deadlock case and we have
564 * no choice but to return an error. For 5.12 and older
565 * kernels returning -ERESTARTSYS will result in the
566 * bdev->bd_mutex being dropped, then reacquired, and
567 * fops->open() being called again. This process can be
568 * repeated safely until both locks are acquired. For 5.13
569 * and newer the -ERESTARTSYS retry logic was removed from
570 * the kernel so the only option is to return the error for
571 * the caller to handle it.
573 if (!mutex_owned(&spa_namespace_lock)) {
574 if (!mutex_tryenter(&spa_namespace_lock)) {
575 mutex_exit(&zv->zv_state_lock);
576 rw_exit(&zv->zv_suspend_lock);
578 #ifdef HAVE_BLKDEV_GET_ERESTARTSYS
580 return (SET_ERROR(-ERESTARTSYS));
582 if ((gethrtime() - start) > timeout)
583 return (SET_ERROR(-ERESTARTSYS));
585 schedule_timeout(MSEC_TO_TICK(10));
589 drop_namespace = B_TRUE;
593 error = -zvol_first_open(zv, !(flag & FMODE_WRITE));
596 mutex_exit(&spa_namespace_lock);
600 if ((flag & FMODE_WRITE) && (zv->zv_flags & ZVOL_RDONLY)) {
601 if (zv->zv_open_count == 0)
604 error = SET_ERROR(-EROFS);
610 mutex_exit(&zv->zv_state_lock);
612 rw_exit(&zv->zv_suspend_lock);
615 zfs_check_media_change(bdev);
621 zvol_release(struct gendisk *disk, fmode_t mode)
624 boolean_t drop_suspend = B_TRUE;
626 rw_enter(&zvol_state_lock, RW_READER);
627 zv = disk->private_data;
629 mutex_enter(&zv->zv_state_lock);
630 ASSERT3U(zv->zv_open_count, >, 0);
632 * make sure zvol is not suspended during last close
633 * (hold zv_suspend_lock) and respect proper lock acquisition
634 * ordering - zv_suspend_lock before zv_state_lock
636 if (zv->zv_open_count == 1) {
637 if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) {
638 mutex_exit(&zv->zv_state_lock);
639 rw_enter(&zv->zv_suspend_lock, RW_READER);
640 mutex_enter(&zv->zv_state_lock);
641 /* check to see if zv_suspend_lock is needed */
642 if (zv->zv_open_count != 1) {
643 rw_exit(&zv->zv_suspend_lock);
644 drop_suspend = B_FALSE;
648 drop_suspend = B_FALSE;
650 rw_exit(&zvol_state_lock);
652 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
655 if (zv->zv_open_count == 0) {
656 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
660 mutex_exit(&zv->zv_state_lock);
663 rw_exit(&zv->zv_suspend_lock);
667 zvol_ioctl(struct block_device *bdev, fmode_t mode,
668 unsigned int cmd, unsigned long arg)
670 zvol_state_t *zv = bdev->bd_disk->private_data;
673 ASSERT3U(zv->zv_open_count, >, 0);
678 invalidate_bdev(bdev);
679 rw_enter(&zv->zv_suspend_lock, RW_READER);
681 if (!(zv->zv_flags & ZVOL_RDONLY))
682 txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0);
684 rw_exit(&zv->zv_suspend_lock);
688 mutex_enter(&zv->zv_state_lock);
689 error = copy_to_user((void *)arg, zv->zv_name, MAXNAMELEN);
690 mutex_exit(&zv->zv_state_lock);
698 return (SET_ERROR(error));
703 zvol_compat_ioctl(struct block_device *bdev, fmode_t mode,
704 unsigned cmd, unsigned long arg)
706 return (zvol_ioctl(bdev, mode, cmd, arg));
709 #define zvol_compat_ioctl NULL
713 zvol_check_events(struct gendisk *disk, unsigned int clearing)
715 unsigned int mask = 0;
717 rw_enter(&zvol_state_lock, RW_READER);
719 zvol_state_t *zv = disk->private_data;
721 mutex_enter(&zv->zv_state_lock);
722 mask = zv->zv_changed ? DISK_EVENT_MEDIA_CHANGE : 0;
724 mutex_exit(&zv->zv_state_lock);
727 rw_exit(&zvol_state_lock);
733 zvol_revalidate_disk(struct gendisk *disk)
735 rw_enter(&zvol_state_lock, RW_READER);
737 zvol_state_t *zv = disk->private_data;
739 mutex_enter(&zv->zv_state_lock);
740 set_capacity(zv->zv_zso->zvo_disk,
741 zv->zv_volsize >> SECTOR_BITS);
742 mutex_exit(&zv->zv_state_lock);
745 rw_exit(&zvol_state_lock);
751 zvol_os_update_volsize(zvol_state_t *zv, uint64_t volsize)
753 struct gendisk *disk = zv->zv_zso->zvo_disk;
755 #if defined(HAVE_REVALIDATE_DISK_SIZE)
756 revalidate_disk_size(disk, zvol_revalidate_disk(disk) == 0);
757 #elif defined(HAVE_REVALIDATE_DISK)
758 revalidate_disk(disk);
760 zvol_revalidate_disk(disk);
766 zvol_os_clear_private(zvol_state_t *zv)
769 * Cleared while holding zvol_state_lock as a writer
770 * which will prevent zvol_open() from opening it.
772 zv->zv_zso->zvo_disk->private_data = NULL;
776 * Provide a simple virtual geometry for legacy compatibility. For devices
777 * smaller than 1 MiB a small head and sector count is used to allow very
778 * tiny devices. For devices over 1 Mib a standard head and sector count
779 * is used to keep the cylinders count reasonable.
782 zvol_getgeo(struct block_device *bdev, struct hd_geometry *geo)
784 zvol_state_t *zv = bdev->bd_disk->private_data;
787 ASSERT3U(zv->zv_open_count, >, 0);
789 sectors = get_capacity(zv->zv_zso->zvo_disk);
791 if (sectors > 2048) {
800 geo->cylinders = sectors / (geo->heads * geo->sectors);
805 static const struct block_device_operations zvol_ops = {
807 .release = zvol_release,
809 .compat_ioctl = zvol_compat_ioctl,
810 .check_events = zvol_check_events,
811 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
812 .revalidate_disk = zvol_revalidate_disk,
814 .getgeo = zvol_getgeo,
815 .owner = THIS_MODULE,
816 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
817 .submit_bio = zvol_submit_bio,
822 * Allocate memory for a new zvol_state_t and setup the required
823 * request queue and generic disk structures for the block device.
825 static zvol_state_t *
826 zvol_alloc(dev_t dev, const char *name)
829 struct zvol_state_os *zso;
832 if (dsl_prop_get_integer(name, "volmode", &volmode, NULL) != 0)
835 if (volmode == ZFS_VOLMODE_DEFAULT)
836 volmode = zvol_volmode;
838 if (volmode == ZFS_VOLMODE_NONE)
841 zv = kmem_zalloc(sizeof (zvol_state_t), KM_SLEEP);
842 zso = kmem_zalloc(sizeof (struct zvol_state_os), KM_SLEEP);
844 zv->zv_volmode = volmode;
846 list_link_init(&zv->zv_next);
847 mutex_init(&zv->zv_state_lock, NULL, MUTEX_DEFAULT, NULL);
849 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
850 #ifdef HAVE_BLK_ALLOC_DISK
851 zso->zvo_disk = blk_alloc_disk(NUMA_NO_NODE);
852 if (zso->zvo_disk == NULL)
855 zso->zvo_disk->minors = ZVOL_MINORS;
856 zso->zvo_queue = zso->zvo_disk->queue;
858 zso->zvo_queue = blk_alloc_queue(NUMA_NO_NODE);
859 if (zso->zvo_queue == NULL)
862 zso->zvo_disk = alloc_disk(ZVOL_MINORS);
863 if (zso->zvo_disk == NULL) {
864 blk_cleanup_queue(zso->zvo_queue);
868 zso->zvo_disk->queue = zso->zvo_queue;
869 #endif /* HAVE_BLK_ALLOC_DISK */
871 zso->zvo_queue = blk_generic_alloc_queue(zvol_request, NUMA_NO_NODE);
872 if (zso->zvo_queue == NULL)
875 zso->zvo_disk = alloc_disk(ZVOL_MINORS);
876 if (zso->zvo_disk == NULL) {
877 blk_cleanup_queue(zso->zvo_queue);
881 zso->zvo_disk->queue = zso->zvo_queue;
882 #endif /* HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
884 blk_queue_set_write_cache(zso->zvo_queue, B_TRUE, B_TRUE);
886 /* Limit read-ahead to a single page to prevent over-prefetching. */
887 blk_queue_set_read_ahead(zso->zvo_queue, 1);
889 /* Disable write merging in favor of the ZIO pipeline. */
890 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, zso->zvo_queue);
892 /* Enable /proc/diskstats */
893 blk_queue_flag_set(QUEUE_FLAG_IO_STAT, zso->zvo_queue);
895 zso->zvo_queue->queuedata = zv;
897 zv->zv_open_count = 0;
898 strlcpy(zv->zv_name, name, MAXNAMELEN);
900 zfs_rangelock_init(&zv->zv_rangelock, NULL, NULL);
901 rw_init(&zv->zv_suspend_lock, NULL, RW_DEFAULT, NULL);
903 zso->zvo_disk->major = zvol_major;
904 zso->zvo_disk->events = DISK_EVENT_MEDIA_CHANGE;
906 if (volmode == ZFS_VOLMODE_DEV) {
908 * ZFS_VOLMODE_DEV disable partitioning on ZVOL devices: set
909 * gendisk->minors = 1 as noted in include/linux/blkdev.h.
910 * Also disable extended partition numbers (GENHD_FL_EXT_DEVT)
911 * and suppresses partition scanning (GENHD_FL_NO_PART_SCAN)
912 * setting gendisk->flags accordingly.
914 zso->zvo_disk->minors = 1;
915 #if defined(GENHD_FL_EXT_DEVT)
916 zso->zvo_disk->flags &= ~GENHD_FL_EXT_DEVT;
918 #if defined(GENHD_FL_NO_PART_SCAN)
919 zso->zvo_disk->flags |= GENHD_FL_NO_PART_SCAN;
922 zso->zvo_disk->first_minor = (dev & MINORMASK);
923 zso->zvo_disk->fops = &zvol_ops;
924 zso->zvo_disk->private_data = zv;
925 snprintf(zso->zvo_disk->disk_name, DISK_NAME_LEN, "%s%d",
926 ZVOL_DEV_NAME, (dev & MINORMASK));
931 kmem_free(zso, sizeof (struct zvol_state_os));
932 kmem_free(zv, sizeof (zvol_state_t));
937 * Cleanup then free a zvol_state_t which was created by zvol_alloc().
938 * At this time, the structure is not opened by anyone, is taken off
939 * the zvol_state_list, and has its private data set to NULL.
940 * The zvol_state_lock is dropped.
942 * This function may take many milliseconds to complete (e.g. we've seen
943 * it take over 256ms), due to the calls to "blk_cleanup_queue" and
944 * "del_gendisk". Thus, consumers need to be careful to account for this
945 * latency when calling this function.
948 zvol_os_free(zvol_state_t *zv)
951 ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
952 ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
953 ASSERT0(zv->zv_open_count);
954 ASSERT3P(zv->zv_zso->zvo_disk->private_data, ==, NULL);
956 rw_destroy(&zv->zv_suspend_lock);
957 zfs_rangelock_fini(&zv->zv_rangelock);
959 del_gendisk(zv->zv_zso->zvo_disk);
960 #if defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS) && \
961 defined(HAVE_BLK_ALLOC_DISK)
962 blk_cleanup_disk(zv->zv_zso->zvo_disk);
964 blk_cleanup_queue(zv->zv_zso->zvo_queue);
965 put_disk(zv->zv_zso->zvo_disk);
968 ida_simple_remove(&zvol_ida,
969 MINOR(zv->zv_zso->zvo_dev) >> ZVOL_MINOR_BITS);
971 mutex_destroy(&zv->zv_state_lock);
972 dataset_kstats_destroy(&zv->zv_kstat);
974 kmem_free(zv->zv_zso, sizeof (struct zvol_state_os));
975 kmem_free(zv, sizeof (zvol_state_t));
979 zvol_wait_close(zvol_state_t *zv)
984 * Create a block device minor node and setup the linkage between it
985 * and the specified volume. Once this function returns the block
986 * device is live and ready for use.
989 zvol_os_create_minor(const char *name)
993 dmu_object_info_t *doi;
999 uint64_t hash = zvol_name_hash(name);
1001 if (zvol_inhibit_dev)
1004 idx = ida_simple_get(&zvol_ida, 0, 0, kmem_flags_convert(KM_SLEEP));
1006 return (SET_ERROR(-idx));
1007 minor = idx << ZVOL_MINOR_BITS;
1009 zv = zvol_find_by_name_hash(name, hash, RW_NONE);
1011 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
1012 mutex_exit(&zv->zv_state_lock);
1013 ida_simple_remove(&zvol_ida, idx);
1014 return (SET_ERROR(EEXIST));
1017 doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP);
1019 error = dmu_objset_own(name, DMU_OST_ZVOL, B_TRUE, B_TRUE, FTAG, &os);
1023 error = dmu_object_info(os, ZVOL_OBJ, doi);
1025 goto out_dmu_objset_disown;
1027 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
1029 goto out_dmu_objset_disown;
1031 zv = zvol_alloc(MKDEV(zvol_major, minor), name);
1033 error = SET_ERROR(EAGAIN);
1034 goto out_dmu_objset_disown;
1038 if (dmu_objset_is_snapshot(os))
1039 zv->zv_flags |= ZVOL_RDONLY;
1041 zv->zv_volblocksize = doi->doi_data_block_size;
1042 zv->zv_volsize = volsize;
1045 set_capacity(zv->zv_zso->zvo_disk, zv->zv_volsize >> 9);
1047 blk_queue_max_hw_sectors(zv->zv_zso->zvo_queue,
1048 (DMU_MAX_ACCESS / 4) >> 9);
1049 blk_queue_max_segments(zv->zv_zso->zvo_queue, UINT16_MAX);
1050 blk_queue_max_segment_size(zv->zv_zso->zvo_queue, UINT_MAX);
1051 blk_queue_physical_block_size(zv->zv_zso->zvo_queue,
1052 zv->zv_volblocksize);
1053 blk_queue_io_opt(zv->zv_zso->zvo_queue, zv->zv_volblocksize);
1054 blk_queue_max_discard_sectors(zv->zv_zso->zvo_queue,
1055 (zvol_max_discard_blocks * zv->zv_volblocksize) >> 9);
1056 blk_queue_discard_granularity(zv->zv_zso->zvo_queue,
1057 zv->zv_volblocksize);
1058 blk_queue_flag_set(QUEUE_FLAG_DISCARD, zv->zv_zso->zvo_queue);
1059 #ifdef QUEUE_FLAG_NONROT
1060 blk_queue_flag_set(QUEUE_FLAG_NONROT, zv->zv_zso->zvo_queue);
1062 #ifdef QUEUE_FLAG_ADD_RANDOM
1063 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, zv->zv_zso->zvo_queue);
1065 /* This flag was introduced in kernel version 4.12. */
1066 #ifdef QUEUE_FLAG_SCSI_PASSTHROUGH
1067 blk_queue_flag_set(QUEUE_FLAG_SCSI_PASSTHROUGH, zv->zv_zso->zvo_queue);
1070 ASSERT3P(zv->zv_zilog, ==, NULL);
1071 zv->zv_zilog = zil_open(os, zvol_get_data);
1072 if (spa_writeable(dmu_objset_spa(os))) {
1073 if (zil_replay_disable)
1074 zil_destroy(zv->zv_zilog, B_FALSE);
1076 zil_replay(os, zv, zvol_replay_vector);
1078 zil_close(zv->zv_zilog);
1079 zv->zv_zilog = NULL;
1080 ASSERT3P(zv->zv_kstat.dk_kstats, ==, NULL);
1081 dataset_kstats_create(&zv->zv_kstat, zv->zv_objset);
1084 * When udev detects the addition of the device it will immediately
1085 * invoke blkid(8) to determine the type of content on the device.
1086 * Prefetching the blocks commonly scanned by blkid(8) will speed
1089 len = MIN(MAX(zvol_prefetch_bytes, 0), SPA_MAXBLOCKSIZE);
1091 dmu_prefetch(os, ZVOL_OBJ, 0, 0, len, ZIO_PRIORITY_SYNC_READ);
1092 dmu_prefetch(os, ZVOL_OBJ, 0, volsize - len, len,
1093 ZIO_PRIORITY_SYNC_READ);
1096 zv->zv_objset = NULL;
1097 out_dmu_objset_disown:
1098 dmu_objset_disown(os, B_TRUE, FTAG);
1100 kmem_free(doi, sizeof (dmu_object_info_t));
1103 * Keep in mind that once add_disk() is called, the zvol is
1104 * announced to the world, and zvol_open()/zvol_release() can
1105 * be called at any time. Incidentally, add_disk() itself calls
1106 * zvol_open()->zvol_first_open() and zvol_release()->zvol_last_close()
1110 rw_enter(&zvol_state_lock, RW_WRITER);
1112 rw_exit(&zvol_state_lock);
1113 #ifdef HAVE_ADD_DISK_RET
1114 error = add_disk(zv->zv_zso->zvo_disk);
1116 add_disk(zv->zv_zso->zvo_disk);
1119 ida_simple_remove(&zvol_ida, idx);
1126 zvol_os_rename_minor(zvol_state_t *zv, const char *newname)
1128 int readonly = get_disk_ro(zv->zv_zso->zvo_disk);
1130 ASSERT(RW_LOCK_HELD(&zvol_state_lock));
1131 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
1133 strlcpy(zv->zv_name, newname, sizeof (zv->zv_name));
1135 /* move to new hashtable entry */
1136 zv->zv_hash = zvol_name_hash(zv->zv_name);
1137 hlist_del(&zv->zv_hlink);
1138 hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash));
1141 * The block device's read-only state is briefly changed causing
1142 * a KOBJ_CHANGE uevent to be issued. This ensures udev detects
1143 * the name change and fixes the symlinks. This does not change
1144 * ZVOL_RDONLY in zv->zv_flags so the actual read-only state never
1145 * changes. This would normally be done using kobject_uevent() but
1146 * that is a GPL-only symbol which is why we need this workaround.
1148 set_disk_ro(zv->zv_zso->zvo_disk, !readonly);
1149 set_disk_ro(zv->zv_zso->zvo_disk, readonly);
1153 zvol_os_set_disk_ro(zvol_state_t *zv, int flags)
1156 set_disk_ro(zv->zv_zso->zvo_disk, flags);
1160 zvol_os_set_capacity(zvol_state_t *zv, uint64_t capacity)
1163 set_capacity(zv->zv_zso->zvo_disk, capacity);
1170 int threads = MIN(MAX(zvol_threads, 1), 1024);
1172 error = register_blkdev(zvol_major, ZVOL_DRIVER);
1174 printk(KERN_INFO "ZFS: register_blkdev() failed %d\n", error);
1177 zvol_taskq = taskq_create(ZVOL_DRIVER, threads, maxclsyspri,
1178 threads * 2, INT_MAX, TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
1179 if (zvol_taskq == NULL) {
1180 unregister_blkdev(zvol_major, ZVOL_DRIVER);
1184 ida_init(&zvol_ida);
1192 unregister_blkdev(zvol_major, ZVOL_DRIVER);
1193 taskq_destroy(zvol_taskq);
1194 ida_destroy(&zvol_ida);
1198 module_param(zvol_inhibit_dev, uint, 0644);
1199 MODULE_PARM_DESC(zvol_inhibit_dev, "Do not create zvol device nodes");
1201 module_param(zvol_major, uint, 0444);
1202 MODULE_PARM_DESC(zvol_major, "Major number for zvol device");
1204 module_param(zvol_threads, uint, 0444);
1205 MODULE_PARM_DESC(zvol_threads, "Max number of threads to handle I/O requests");
1207 module_param(zvol_request_sync, uint, 0644);
1208 MODULE_PARM_DESC(zvol_request_sync, "Synchronously handle bio requests");
1210 module_param(zvol_max_discard_blocks, ulong, 0444);
1211 MODULE_PARM_DESC(zvol_max_discard_blocks, "Max number of blocks to discard");
1213 module_param(zvol_prefetch_bytes, uint, 0644);
1214 MODULE_PARM_DESC(zvol_prefetch_bytes, "Prefetch N bytes at zvol start+end");
1216 module_param(zvol_volmode, uint, 0644);
1217 MODULE_PARM_DESC(zvol_volmode, "Default volmode property value");