4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2016 Actifio, Inc. All rights reserved.
34 #include <sys/signal.h>
37 #include <sys/processor.h>
38 #include <sys/zfs_context.h>
39 #include <sys/rrwlock.h>
40 #include <sys/utsname.h>
42 #include <sys/systeminfo.h>
43 #include <zfs_fletcher.h>
46 * Emulation of kernel services in userland.
51 vnode_t *rootdir = (vnode_t *)0xabcd1234;
52 char hw_serial[HW_HOSTID_LEN];
53 struct utsname hw_utsname;
54 vmem_t *zio_arena = NULL;
56 /* If set, all blocks read will be copied to the specified directory. */
57 char *vn_dumpdir = NULL;
59 /* this only exists to have its address taken */
63 * =========================================================================
65 * =========================================================================
68 pthread_cond_t kthread_cond = PTHREAD_COND_INITIALIZER;
69 pthread_mutex_t kthread_lock = PTHREAD_MUTEX_INITIALIZER;
70 pthread_key_t kthread_key;
78 VERIFY3S(pthread_key_create(&kthread_key, NULL), ==, 0);
80 /* Create entry for primary kthread */
81 kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
82 kt->t_tid = pthread_self();
85 VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
87 /* Only the main thread should be running at the moment */
88 ASSERT3S(kthread_nr, ==, 0);
95 kthread_t *kt = curthread;
97 ASSERT(pthread_equal(kt->t_tid, pthread_self()));
98 ASSERT3P(kt->t_func, ==, NULL);
100 umem_free(kt, sizeof (kthread_t));
102 /* Wait for all threads to exit via thread_exit() */
103 VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
105 kthread_nr--; /* Main thread is exiting */
107 while (kthread_nr > 0)
108 VERIFY0(pthread_cond_wait(&kthread_cond, &kthread_lock));
110 ASSERT3S(kthread_nr, ==, 0);
111 VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
113 VERIFY3S(pthread_key_delete(kthread_key), ==, 0);
117 zk_thread_current(void)
119 kthread_t *kt = pthread_getspecific(kthread_key);
121 ASSERT3P(kt, !=, NULL);
127 zk_thread_helper(void *arg)
129 kthread_t *kt = (kthread_t *) arg;
131 VERIFY3S(pthread_setspecific(kthread_key, kt), ==, 0);
133 VERIFY3S(pthread_mutex_lock(&kthread_lock), ==, 0);
135 VERIFY3S(pthread_mutex_unlock(&kthread_lock), ==, 0);
136 (void) setpriority(PRIO_PROCESS, 0, kt->t_pri);
138 kt->t_tid = pthread_self();
139 ((thread_func_arg_t) kt->t_func)(kt->t_arg);
141 /* Unreachable, thread must exit with thread_exit() */
148 zk_thread_create(caddr_t stk, size_t stksize, thread_func_t func, void *arg,
149 size_t len, proc_t *pp, int state, pri_t pri, int detachstate)
155 ASSERT0(state & ~TS_RUN);
157 kt = umem_zalloc(sizeof (kthread_t), UMEM_NOFAIL);
162 VERIFY0(pthread_attr_init(&attr));
163 VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
166 * We allow the default stack size in user space to be specified by
167 * setting the ZFS_STACK_SIZE environment variable. This allows us
168 * the convenience of observing and debugging stack overruns in
169 * user space. Explicitly specified stack sizes will be honored.
170 * The usage of ZFS_STACK_SIZE is discussed further in the
171 * ENVIRONMENT VARIABLES sections of the ztest(1) man page.
174 stkstr = getenv("ZFS_STACK_SIZE");
177 stksize = TS_STACK_MAX;
179 stksize = MAX(atoi(stkstr), TS_STACK_MIN);
182 VERIFY3S(stksize, >, 0);
183 stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
185 * If this ever fails, it may be because the stack size is not a
186 * multiple of system page size.
188 VERIFY0(pthread_attr_setstacksize(&attr, stksize));
189 VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
191 VERIFY0(pthread_create(&kt->t_tid, &attr, &zk_thread_helper, kt));
192 VERIFY0(pthread_attr_destroy(&attr));
200 kthread_t *kt = curthread;
202 ASSERT(pthread_equal(kt->t_tid, pthread_self()));
204 umem_free(kt, sizeof (kthread_t));
206 VERIFY0(pthread_mutex_lock(&kthread_lock));
208 VERIFY0(pthread_mutex_unlock(&kthread_lock));
210 VERIFY0(pthread_cond_broadcast(&kthread_cond));
211 pthread_exit((void *)TS_MAGIC);
215 zk_thread_join(kt_did_t tid)
219 pthread_join((pthread_t)tid, &ret);
220 VERIFY3P(ret, ==, (void *)TS_MAGIC);
224 * =========================================================================
226 * =========================================================================
230 kstat_create(const char *module, int instance, const char *name,
231 const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag)
238 kstat_install(kstat_t *ksp)
243 kstat_delete(kstat_t *ksp)
248 kstat_waitq_enter(kstat_io_t *kiop)
253 kstat_waitq_exit(kstat_io_t *kiop)
258 kstat_runq_enter(kstat_io_t *kiop)
263 kstat_runq_exit(kstat_io_t *kiop)
268 kstat_waitq_to_runq(kstat_io_t *kiop)
273 kstat_runq_back_to_waitq(kstat_io_t *kiop)
277 kstat_set_raw_ops(kstat_t *ksp,
278 int (*headers)(char *buf, size_t size),
279 int (*data)(char *buf, size_t size, void *data),
280 void *(*addr)(kstat_t *ksp, loff_t index))
284 * =========================================================================
286 * =========================================================================
290 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
292 ASSERT3S(type, ==, MUTEX_DEFAULT);
293 ASSERT3P(cookie, ==, NULL);
294 mp->m_owner = MTX_INIT;
295 mp->m_magic = MTX_MAGIC;
296 VERIFY3S(pthread_mutex_init(&mp->m_lock, NULL), ==, 0);
300 mutex_destroy(kmutex_t *mp)
302 ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
303 ASSERT3P(mp->m_owner, ==, MTX_INIT);
304 ASSERT0(pthread_mutex_destroy(&(mp)->m_lock));
305 mp->m_owner = MTX_DEST;
310 mutex_enter(kmutex_t *mp)
312 ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
313 ASSERT3P(mp->m_owner, !=, MTX_DEST);
314 ASSERT3P(mp->m_owner, !=, curthread);
315 VERIFY3S(pthread_mutex_lock(&mp->m_lock), ==, 0);
316 ASSERT3P(mp->m_owner, ==, MTX_INIT);
317 mp->m_owner = curthread;
321 mutex_tryenter(kmutex_t *mp)
324 ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
325 ASSERT3P(mp->m_owner, !=, MTX_DEST);
326 if (0 == (err = pthread_mutex_trylock(&mp->m_lock))) {
327 ASSERT3P(mp->m_owner, ==, MTX_INIT);
328 mp->m_owner = curthread;
331 VERIFY3S(err, ==, EBUSY);
337 mutex_exit(kmutex_t *mp)
339 ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
340 ASSERT3P(mutex_owner(mp), ==, curthread);
341 mp->m_owner = MTX_INIT;
342 VERIFY3S(pthread_mutex_unlock(&mp->m_lock), ==, 0);
346 mutex_owner(kmutex_t *mp)
348 ASSERT3U(mp->m_magic, ==, MTX_MAGIC);
349 return (mp->m_owner);
353 mutex_held(kmutex_t *mp)
355 return (mp->m_owner == curthread);
359 * =========================================================================
361 * =========================================================================
365 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
367 ASSERT3S(type, ==, RW_DEFAULT);
368 ASSERT3P(arg, ==, NULL);
369 VERIFY3S(pthread_rwlock_init(&rwlp->rw_lock, NULL), ==, 0);
370 rwlp->rw_owner = RW_INIT;
371 rwlp->rw_wr_owner = RW_INIT;
372 rwlp->rw_readers = 0;
373 rwlp->rw_magic = RW_MAGIC;
377 rw_destroy(krwlock_t *rwlp)
379 ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
380 ASSERT(rwlp->rw_readers == 0 && rwlp->rw_wr_owner == RW_INIT);
381 VERIFY3S(pthread_rwlock_destroy(&rwlp->rw_lock), ==, 0);
386 rw_enter(krwlock_t *rwlp, krw_t rw)
388 ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
389 ASSERT3P(rwlp->rw_owner, !=, curthread);
390 ASSERT3P(rwlp->rw_wr_owner, !=, curthread);
392 if (rw == RW_READER) {
393 VERIFY3S(pthread_rwlock_rdlock(&rwlp->rw_lock), ==, 0);
394 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
396 atomic_inc_uint(&rwlp->rw_readers);
398 VERIFY3S(pthread_rwlock_wrlock(&rwlp->rw_lock), ==, 0);
399 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
400 ASSERT3U(rwlp->rw_readers, ==, 0);
402 rwlp->rw_wr_owner = curthread;
405 rwlp->rw_owner = curthread;
409 rw_exit(krwlock_t *rwlp)
411 ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
412 ASSERT(RW_LOCK_HELD(rwlp));
414 if (RW_READ_HELD(rwlp))
415 atomic_dec_uint(&rwlp->rw_readers);
417 rwlp->rw_wr_owner = RW_INIT;
419 rwlp->rw_owner = RW_INIT;
420 VERIFY3S(pthread_rwlock_unlock(&rwlp->rw_lock), ==, 0);
424 rw_tryenter(krwlock_t *rwlp, krw_t rw)
428 ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
431 rv = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
433 rv = pthread_rwlock_trywrlock(&rwlp->rw_lock);
436 ASSERT3P(rwlp->rw_wr_owner, ==, RW_INIT);
439 atomic_inc_uint(&rwlp->rw_readers);
441 ASSERT3U(rwlp->rw_readers, ==, 0);
442 rwlp->rw_wr_owner = curthread;
445 rwlp->rw_owner = curthread;
449 VERIFY3S(rv, ==, EBUSY);
455 rw_tryupgrade(krwlock_t *rwlp)
457 ASSERT3U(rwlp->rw_magic, ==, RW_MAGIC);
463 * =========================================================================
464 * condition variables
465 * =========================================================================
469 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
471 ASSERT3S(type, ==, CV_DEFAULT);
472 cv->cv_magic = CV_MAGIC;
473 VERIFY0(pthread_cond_init(&cv->cv, NULL));
477 cv_destroy(kcondvar_t *cv)
479 ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
480 VERIFY0(pthread_cond_destroy(&cv->cv));
485 cv_wait(kcondvar_t *cv, kmutex_t *mp)
487 ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
488 ASSERT3P(mutex_owner(mp), ==, curthread);
489 mp->m_owner = MTX_INIT;
490 VERIFY0(pthread_cond_wait(&cv->cv, &mp->m_lock));
491 mp->m_owner = curthread;
495 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
502 ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
504 delta = abstime - ddi_get_lbolt();
508 VERIFY(gettimeofday(&tv, NULL) == 0);
510 ts.tv_sec = tv.tv_sec + delta / hz;
511 ts.tv_nsec = tv.tv_usec * 1000 + (delta % hz) * (NANOSEC / hz);
512 if (ts.tv_nsec >= NANOSEC) {
514 ts.tv_nsec -= NANOSEC;
517 ASSERT3P(mutex_owner(mp), ==, curthread);
518 mp->m_owner = MTX_INIT;
519 error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
520 mp->m_owner = curthread;
522 if (error == ETIMEDOUT)
532 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
539 ASSERT(flag == 0 || flag == CALLOUT_FLAG_ABSOLUTE);
542 if (flag & CALLOUT_FLAG_ABSOLUTE)
543 delta -= gethrtime();
548 ts.tv_sec = delta / NANOSEC;
549 ts.tv_nsec = delta % NANOSEC;
551 ASSERT(mutex_owner(mp) == curthread);
553 error = pthread_cond_timedwait(&cv->cv, &mp->m_lock, &ts);
554 mp->m_owner = curthread;
556 if (error == ETIMEDOUT)
565 cv_signal(kcondvar_t *cv)
567 ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
568 VERIFY0(pthread_cond_signal(&cv->cv));
572 cv_broadcast(kcondvar_t *cv)
574 ASSERT3U(cv->cv_magic, ==, CV_MAGIC);
575 VERIFY0(pthread_cond_broadcast(&cv->cv));
579 * =========================================================================
581 * =========================================================================
584 * Note: for the xxxat() versions of these functions, we assume that the
585 * starting vp is always rootdir (which is true for spa_directory.c, the only
586 * ZFS consumer of these interfaces). We assert this is true, and then emulate
587 * them by adding '/' in front of the path.
592 vn_open(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2, int x3)
602 realpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
605 * If we're accessing a real disk from userland, we need to use
606 * the character interface to avoid caching. This is particularly
607 * important if we're trying to look at a real in-kernel storage
608 * pool from userland, e.g. via zdb, because otherwise we won't
609 * see the changes occurring under the segmap cache.
610 * On the other hand, the stupid character device returns zero
611 * for its size. So -- gag -- we open the block device to get
612 * its size, and remember it for subsequent VOP_GETATTR().
614 #if defined(__sun__) || defined(__sun)
615 if (strncmp(path, "/dev/", 5) == 0) {
620 fd = open64(path, O_RDONLY);
626 if (fstat64(fd, &st) == -1) {
633 (void) sprintf(realpath, "%s", path);
634 dsk = strstr(path, "/dsk/");
636 (void) sprintf(realpath + (dsk - path) + 1, "r%s",
639 (void) sprintf(realpath, "%s", path);
640 if (!(flags & FCREAT) && stat64(realpath, &st) == -1) {
647 if (!(flags & FCREAT) && S_ISBLK(st.st_mode)) {
651 /* We shouldn't be writing to block devices in userspace */
652 VERIFY(!(flags & FWRITE));
656 old_umask = umask(0);
659 * The construct 'flags - FREAD' conveniently maps combinations of
660 * FREAD and FWRITE to the corresponding O_RDONLY, O_WRONLY, and O_RDWR.
662 fd = open64(realpath, flags - FREAD, mode);
666 (void) umask(old_umask);
668 if (vn_dumpdir != NULL) {
669 char *dumppath = umem_zalloc(MAXPATHLEN, UMEM_NOFAIL);
670 (void) snprintf(dumppath, MAXPATHLEN,
671 "%s/%s", vn_dumpdir, basename(realpath));
672 dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666);
673 umem_free(dumppath, MAXPATHLEN);
689 if (fstat64_blk(fd, &st) == -1) {
695 (void) fcntl(fd, F_SETFD, FD_CLOEXEC);
697 *vpp = vp = umem_zalloc(sizeof (vnode_t), UMEM_NOFAIL);
700 vp->v_size = st.st_size;
701 vp->v_path = spa_strdup(path);
702 vp->v_dump_fd = dump_fd;
709 vn_openat(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2,
710 int x3, vnode_t *startvp, int fd)
712 char *realpath = umem_alloc(strlen(path) + 2, UMEM_NOFAIL);
715 ASSERT(startvp == rootdir);
716 (void) sprintf(realpath, "/%s", path);
718 /* fd ignored for now, need if want to simulate nbmand support */
719 ret = vn_open(realpath, x1, flags, mode, vpp, x2, x3);
721 umem_free(realpath, strlen(path) + 2);
728 vn_rdwr(int uio, vnode_t *vp, void *addr, ssize_t len, offset_t offset,
729 int x1, int x2, rlim64_t x3, void *x4, ssize_t *residp)
731 ssize_t rc, done = 0, split;
733 if (uio == UIO_READ) {
734 rc = pread64(vp->v_fd, addr, len, offset);
735 if (vp->v_dump_fd != -1) {
737 status = pwrite64(vp->v_dump_fd, addr, rc, offset);
738 ASSERT(status != -1);
742 * To simulate partial disk writes, we split writes into two
743 * system calls so that the process can be killed in between.
745 int sectors = len >> SPA_MINBLOCKSHIFT;
746 split = (sectors > 0 ? rand() % sectors : 0) <<
748 rc = pwrite64(vp->v_fd, addr, split, offset);
751 rc = pwrite64(vp->v_fd, (char *)addr + split,
752 len - split, offset + split);
757 if (rc == -1 && errno == EINVAL) {
759 * Under Linux, this most likely means an alignment issue
760 * (memory or disk) due to O_DIRECT, so we abort() in order to
761 * catch the offender.
772 *residp = len - done;
773 else if (done != len)
779 vn_close(vnode_t *vp)
782 if (vp->v_dump_fd != -1)
783 close(vp->v_dump_fd);
784 spa_strfree(vp->v_path);
785 umem_free(vp, sizeof (vnode_t));
789 * At a minimum we need to update the size since vdev_reopen()
790 * will no longer call vn_openat().
793 fop_getattr(vnode_t *vp, vattr_t *vap)
798 if (fstat64_blk(vp->v_fd, &st) == -1) {
804 vap->va_size = st.st_size;
809 * =========================================================================
810 * Figure out which debugging statements to print
811 * =========================================================================
814 static char *dprintf_string;
815 static int dprintf_print_all;
818 dprintf_find_string(const char *string)
820 char *tmp_str = dprintf_string;
821 int len = strlen(string);
824 * Find out if this is a string we want to print.
825 * String format: file1.c,function_name1,file2.c,file3.c
828 while (tmp_str != NULL) {
829 if (strncmp(tmp_str, string, len) == 0 &&
830 (tmp_str[len] == ',' || tmp_str[len] == '\0'))
832 tmp_str = strchr(tmp_str, ',');
834 tmp_str++; /* Get rid of , */
840 dprintf_setup(int *argc, char **argv)
845 * Debugging can be specified two ways: by setting the
846 * environment variable ZFS_DEBUG, or by including a
847 * "debug=..." argument on the command line. The command
848 * line setting overrides the environment variable.
851 for (i = 1; i < *argc; i++) {
852 int len = strlen("debug=");
853 /* First look for a command line argument */
854 if (strncmp("debug=", argv[i], len) == 0) {
855 dprintf_string = argv[i] + len;
856 /* Remove from args */
857 for (j = i; j < *argc; j++)
864 if (dprintf_string == NULL) {
865 /* Look for ZFS_DEBUG environment variable */
866 dprintf_string = getenv("ZFS_DEBUG");
870 * Are we just turning on all debugging?
872 if (dprintf_find_string("on"))
873 dprintf_print_all = 1;
875 if (dprintf_string != NULL)
876 zfs_flags |= ZFS_DEBUG_DPRINTF;
880 * =========================================================================
882 * =========================================================================
885 __dprintf(const char *file, const char *func, int line, const char *fmt, ...)
891 * Get rid of annoying "../common/" prefix to filename.
893 newfile = strrchr(file, '/');
894 if (newfile != NULL) {
895 newfile = newfile + 1; /* Get rid of leading / */
900 if (dprintf_print_all ||
901 dprintf_find_string(newfile) ||
902 dprintf_find_string(func)) {
903 /* Print out just the function name if requested */
905 if (dprintf_find_string("pid"))
906 (void) printf("%d ", getpid());
907 if (dprintf_find_string("tid"))
908 (void) printf("%u ", (uint_t) pthread_self());
909 if (dprintf_find_string("cpu"))
910 (void) printf("%u ", getcpuid());
911 if (dprintf_find_string("time"))
912 (void) printf("%llu ", gethrtime());
913 if (dprintf_find_string("long"))
914 (void) printf("%s, line %d: ", newfile, line);
915 (void) printf("%s: ", func);
917 (void) vprintf(fmt, adx);
924 * =========================================================================
925 * cmn_err() and panic()
926 * =========================================================================
928 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
929 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
932 vpanic(const char *fmt, va_list adx)
934 (void) fprintf(stderr, "error: ");
935 (void) vfprintf(stderr, fmt, adx);
936 (void) fprintf(stderr, "\n");
938 abort(); /* think of it as a "user-level crash dump" */
942 panic(const char *fmt, ...)
952 vcmn_err(int ce, const char *fmt, va_list adx)
956 if (ce != CE_NOTE) { /* suppress noise in userland stress testing */
957 (void) fprintf(stderr, "%s", ce_prefix[ce]);
958 (void) vfprintf(stderr, fmt, adx);
959 (void) fprintf(stderr, "%s", ce_suffix[ce]);
965 cmn_err(int ce, const char *fmt, ...)
970 vcmn_err(ce, fmt, adx);
975 * =========================================================================
977 * =========================================================================
980 kobj_open_file(char *name)
985 /* set vp as the _fd field of the file */
986 if (vn_openat(name, UIO_SYSSPACE, FREAD, 0, &vp, 0, 0, rootdir,
988 return ((void *)-1UL);
990 file = umem_zalloc(sizeof (struct _buf), UMEM_NOFAIL);
991 file->_fd = (intptr_t)vp;
996 kobj_read_file(struct _buf *file, char *buf, unsigned size, unsigned off)
1000 if (vn_rdwr(UIO_READ, (vnode_t *)file->_fd, buf, size, (offset_t)off,
1001 UIO_SYSSPACE, 0, 0, 0, &resid) != 0)
1004 return (size - resid);
1008 kobj_close_file(struct _buf *file)
1010 vn_close((vnode_t *)file->_fd);
1011 umem_free(file, sizeof (struct _buf));
1015 kobj_get_filesize(struct _buf *file, uint64_t *size)
1018 vnode_t *vp = (vnode_t *)file->_fd;
1020 if (fstat64(vp->v_fd, &st) == -1) {
1029 * =========================================================================
1031 * =========================================================================
1035 delay(clock_t ticks)
1037 poll(0, 0, ticks * (1000 / hz));
1041 * Find highest one bit set.
1042 * Returns bit number + 1 of highest bit that is set, otherwise returns 0.
1043 * High order bit is 31 (or 63 in _LP64 kernel).
1046 highbit64(uint64_t i)
1052 if (i & 0xffffffff00000000ULL) {
1055 if (i & 0xffff0000) {
1074 * Find lowest one bit set.
1075 * Returns bit number + 1 of lowest bit that is set, otherwise returns 0.
1076 * This is basically a reimplementation of ffsll(), which is GNU specific.
1079 lowbit64(uint64_t i)
1081 register int h = 64;
1085 if (i & 0x00000000ffffffffULL)
1117 static int random_fd = -1, urandom_fd = -1;
1120 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
1127 while (resid != 0) {
1128 bytes = read(fd, ptr, resid);
1129 ASSERT3S(bytes, >=, 0);
1138 random_get_bytes(uint8_t *ptr, size_t len)
1140 return (random_get_bytes_common(ptr, len, random_fd));
1144 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
1146 return (random_get_bytes_common(ptr, len, urandom_fd));
1150 ddi_strtoul(const char *hw_serial, char **nptr, int base, unsigned long *result)
1154 *result = strtoul(hw_serial, &end, base);
1161 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result)
1165 *result = strtoull(str, &end, base);
1174 return (&hw_utsname);
1178 * =========================================================================
1179 * kernel emulation setup & teardown
1180 * =========================================================================
1183 umem_out_of_memory(void)
1185 char errmsg[] = "out of memory -- generating core dump\n";
1187 (void) fprintf(stderr, "%s", errmsg);
1192 static unsigned long
1193 get_spl_hostid(void)
1196 unsigned long hostid;
1198 f = fopen("/sys/module/spl/parameters/spl_hostid", "r");
1201 if (fscanf(f, "%lu", &hostid) != 1)
1204 return (hostid & 0xffffffff);
1208 get_system_hostid(void)
1210 unsigned long system_hostid = get_spl_hostid();
1211 if (system_hostid == 0)
1212 system_hostid = gethostid() & 0xffffffff;
1213 return (system_hostid);
1217 kernel_init(int mode)
1219 extern uint_t rrw_tsd_key;
1221 umem_nofail_callback(umem_out_of_memory);
1223 physmem = sysconf(_SC_PHYS_PAGES);
1225 dprintf("physmem = %llu pages (%.2f GB)\n", physmem,
1226 (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
1228 (void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
1229 (mode & FWRITE) ? get_system_hostid() : 0);
1231 VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1);
1232 VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1);
1233 VERIFY0(uname(&hw_utsname));
1236 system_taskq_init();
1242 tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
1251 system_taskq_fini();
1262 crgetuid(cred_t *cr)
1268 crgetruid(cred_t *cr)
1274 crgetgid(cred_t *cr)
1280 crgetngroups(cred_t *cr)
1286 crgetgroups(cred_t *cr)
1292 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr)
1298 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr)
1304 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr)
1310 secpolicy_zfs(const cred_t *cr)
1316 ksid_lookupdomain(const char *dom)
1320 kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL);
1321 kd->kd_name = spa_strdup(dom);
1326 ksiddomain_rele(ksiddomain_t *ksid)
1328 spa_strfree(ksid->kd_name);
1329 umem_free(ksid, sizeof (ksiddomain_t));
1333 kmem_vasprintf(const char *fmt, va_list adx)
1338 va_copy(adx_copy, adx);
1339 VERIFY(vasprintf(&buf, fmt, adx_copy) != -1);
1346 kmem_asprintf(const char *fmt, ...)
1352 VERIFY(vasprintf(&buf, fmt, adx) != -1);
1360 zfs_onexit_fd_hold(int fd, minor_t *minorp)
1368 zfs_onexit_fd_rele(int fd)
1374 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
1375 uint64_t *action_handle)
1382 zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire)
1389 zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data)
1395 spl_fstrans_mark(void)
1397 return ((fstrans_cookie_t) 0);
1401 spl_fstrans_unmark(fstrans_cookie_t cookie)
1406 spl_fstrans_check(void)
1411 void *zvol_tag = "zvol_tag";
1414 zvol_create_minors(spa_t *spa, const char *name, boolean_t async)
1419 zvol_remove_minor(spa_t *spa, const char *name, boolean_t async)
1424 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async)
1429 zvol_rename_minors(spa_t *spa, const char *oldname, const char *newname,