4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
27 * Copyright 2013 Saso Kiselkov. All rights reserved.
31 * Copyright (c) 2016 by Delphix. All rights reserved.
38 * ZFS's 2nd and 4th order Fletcher checksums are defined by the following
39 * recurrence relations:
47 * c = c + b (fletcher-4 only)
50 * d = d + c (fletcher-4 only)
54 * a_0 = b_0 = c_0 = d_0 = 0
56 * f_0 .. f_(n-1) are the input data.
58 * Using standard techniques, these translate into the following series:
63 * n /___| n - i n /___| n - i
68 * \ | i*(i+1) \ | i*(i+1)*(i+2)
69 * c = > ------- f d = > ------------- f
70 * n /___| 2 n - i n /___| 6 n - i
73 * For fletcher-2, the f_is are 64-bit, and [ab]_i are 64-bit accumulators.
74 * Since the additions are done mod (2^64), errors in the high bits may not
75 * be noticed. For this reason, fletcher-2 is deprecated.
77 * For fletcher-4, the f_is are 32-bit, and [abcd]_i are 64-bit accumulators.
78 * A conservative estimate of how big the buffer can get before we overflow
79 * can be estimated using f_i = 0xffffffff for all i:
82 * f=2^32-1;d=0; for (i = 1; d<2^64; i++) { d += f*i*(i+1)*(i+2)/6 }; (i-1)*4
87 * So blocks of up to 2k will not overflow. Our largest block size is
88 * 128k, which has 32k 4-byte words, so we can compute the largest possible
89 * accumulators, then divide by 2^64 to figure the max amount of overflow:
92 * a=b=c=d=0; f=2^32-1; for (i=1; i<=32*1024; i++) { a+=f; b+=a; c+=b; d+=c }
93 * a/2^64;b/2^64;c/2^64;d/2^64
101 * So a and b cannot overflow. To make sure each bit of input has some
102 * effect on the contents of c and d, we can look at what the factors of
103 * the coefficients in the equations for c_n and d_n are. The number of 2s
104 * in the factors determines the lowest set bit in the multiplier. Running
105 * through the cases for n*(n+1)/2 reveals that the highest power of 2 is
106 * 2^14, and for n*(n+1)*(n+2)/6 it is 2^15. So while some data may overflow
107 * the 64-bit accumulators, every bit of every f_i effects every accumulator,
108 * even for 128k blocks.
110 * If we wanted to make a stronger version of fletcher4 (fletcher4c?),
111 * we could do our calculations mod (2^32 - 1) by adding in the carries
112 * periodically, and store the number of carries in the top 32-bits.
114 * --------------------
115 * Checksum Performance
116 * --------------------
118 * There are two interesting components to checksum performance: cached and
119 * uncached performance. With cached data, fletcher-2 is about four times
120 * faster than fletcher-4. With uncached data, the performance difference is
121 * negligible, since the cost of a cache fill dominates the processing time.
122 * Even though fletcher-4 is slower than fletcher-2, it is still a pretty
123 * efficient pass over the data.
125 * In normal operation, the data which is being checksummed is in a buffer
126 * which has been filled either by:
128 * 1. a compression step, which will be mostly cached, or
129 * 2. a bcopy() or copyin(), which will be uncached (because the
130 * copy is cache-bypassing).
132 * For both cached and uncached data, both fletcher checksums are much faster
133 * than sha-256, and slower than 'off', which doesn't touch the data at all.
136 #include <sys/types.h>
137 #include <sys/sysmacros.h>
138 #include <sys/byteorder.h>
140 #include <sys/simd.h>
141 #include <sys/zio_checksum.h>
142 #include <sys/zfs_context.h>
143 #include <zfs_fletcher.h>
145 #define FLETCHER_MIN_SIMD_SIZE 64
147 static void fletcher_4_scalar_init(fletcher_4_ctx_t *ctx);
148 static void fletcher_4_scalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp);
149 static void fletcher_4_scalar_native(fletcher_4_ctx_t *ctx,
150 const void *buf, uint64_t size);
151 static void fletcher_4_scalar_byteswap(fletcher_4_ctx_t *ctx,
152 const void *buf, uint64_t size);
153 static boolean_t fletcher_4_scalar_valid(void);
155 static const fletcher_4_ops_t fletcher_4_scalar_ops = {
156 .init_native = fletcher_4_scalar_init,
157 .fini_native = fletcher_4_scalar_fini,
158 .compute_native = fletcher_4_scalar_native,
159 .init_byteswap = fletcher_4_scalar_init,
160 .fini_byteswap = fletcher_4_scalar_fini,
161 .compute_byteswap = fletcher_4_scalar_byteswap,
162 .valid = fletcher_4_scalar_valid,
166 static fletcher_4_ops_t fletcher_4_fastest_impl = {
168 .valid = fletcher_4_scalar_valid
171 static const fletcher_4_ops_t *fletcher_4_impls[] = {
172 &fletcher_4_scalar_ops,
173 &fletcher_4_superscalar_ops,
174 &fletcher_4_superscalar4_ops,
175 #if defined(HAVE_SSE2)
176 &fletcher_4_sse2_ops,
178 #if defined(HAVE_SSE2) && defined(HAVE_SSSE3)
179 &fletcher_4_ssse3_ops,
181 #if defined(HAVE_AVX) && defined(HAVE_AVX2)
182 &fletcher_4_avx2_ops,
184 #if defined(__x86_64) && defined(HAVE_AVX512F)
185 &fletcher_4_avx512f_ops,
187 #if defined(__x86_64) && defined(HAVE_AVX512BW)
188 &fletcher_4_avx512bw_ops,
190 #if defined(__aarch64__) && !defined(__FreeBSD__)
191 &fletcher_4_aarch64_neon_ops,
195 /* Hold all supported implementations */
196 static uint32_t fletcher_4_supp_impls_cnt = 0;
197 static fletcher_4_ops_t *fletcher_4_supp_impls[ARRAY_SIZE(fletcher_4_impls)];
199 /* Select fletcher4 implementation */
200 #define IMPL_FASTEST (UINT32_MAX)
201 #define IMPL_CYCLE (UINT32_MAX - 1)
202 #define IMPL_SCALAR (0)
204 static uint32_t fletcher_4_impl_chosen = IMPL_FASTEST;
206 #define IMPL_READ(i) (*(volatile uint32_t *) &(i))
208 static struct fletcher_4_impl_selector {
209 const char *fis_name;
211 } fletcher_4_impl_selectors[] = {
212 { "cycle", IMPL_CYCLE },
213 { "fastest", IMPL_FASTEST },
214 { "scalar", IMPL_SCALAR }
218 static kstat_t *fletcher_4_kstat;
220 static struct fletcher_4_kstat {
223 } fletcher_4_stat_data[ARRAY_SIZE(fletcher_4_impls) + 1];
226 /* Indicate that benchmark has been completed */
227 static boolean_t fletcher_4_initialized = B_FALSE;
230 fletcher_init(zio_cksum_t *zcp)
232 ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
236 fletcher_2_incremental_native(void *buf, size_t size, void *data)
238 zio_cksum_t *zcp = data;
240 const uint64_t *ip = buf;
241 const uint64_t *ipend = ip + (size / sizeof (uint64_t));
242 uint64_t a0, b0, a1, b1;
244 a0 = zcp->zc_word[0];
245 a1 = zcp->zc_word[1];
246 b0 = zcp->zc_word[2];
247 b1 = zcp->zc_word[3];
249 for (; ip < ipend; ip += 2) {
256 ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
261 fletcher_2_native(const void *buf, uint64_t size,
262 const void *ctx_template, zio_cksum_t *zcp)
266 (void) fletcher_2_incremental_native((void *) buf, size, zcp);
270 fletcher_2_incremental_byteswap(void *buf, size_t size, void *data)
272 zio_cksum_t *zcp = data;
274 const uint64_t *ip = buf;
275 const uint64_t *ipend = ip + (size / sizeof (uint64_t));
276 uint64_t a0, b0, a1, b1;
278 a0 = zcp->zc_word[0];
279 a1 = zcp->zc_word[1];
280 b0 = zcp->zc_word[2];
281 b1 = zcp->zc_word[3];
283 for (; ip < ipend; ip += 2) {
284 a0 += BSWAP_64(ip[0]);
285 a1 += BSWAP_64(ip[1]);
290 ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1);
295 fletcher_2_byteswap(const void *buf, uint64_t size,
296 const void *ctx_template, zio_cksum_t *zcp)
300 (void) fletcher_2_incremental_byteswap((void *) buf, size, zcp);
303 ZFS_NO_SANITIZE_UNDEFINED
305 fletcher_4_scalar_init(fletcher_4_ctx_t *ctx)
307 ZIO_SET_CHECKSUM(&ctx->scalar, 0, 0, 0, 0);
310 ZFS_NO_SANITIZE_UNDEFINED
312 fletcher_4_scalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp)
314 memcpy(zcp, &ctx->scalar, sizeof (zio_cksum_t));
317 ZFS_NO_SANITIZE_UNDEFINED
319 fletcher_4_scalar_native(fletcher_4_ctx_t *ctx, const void *buf,
322 const uint32_t *ip = buf;
323 const uint32_t *ipend = ip + (size / sizeof (uint32_t));
326 a = ctx->scalar.zc_word[0];
327 b = ctx->scalar.zc_word[1];
328 c = ctx->scalar.zc_word[2];
329 d = ctx->scalar.zc_word[3];
331 for (; ip < ipend; ip++) {
338 ZIO_SET_CHECKSUM(&ctx->scalar, a, b, c, d);
341 ZFS_NO_SANITIZE_UNDEFINED
343 fletcher_4_scalar_byteswap(fletcher_4_ctx_t *ctx, const void *buf,
346 const uint32_t *ip = buf;
347 const uint32_t *ipend = ip + (size / sizeof (uint32_t));
350 a = ctx->scalar.zc_word[0];
351 b = ctx->scalar.zc_word[1];
352 c = ctx->scalar.zc_word[2];
353 d = ctx->scalar.zc_word[3];
355 for (; ip < ipend; ip++) {
356 a += BSWAP_32(ip[0]);
362 ZIO_SET_CHECKSUM(&ctx->scalar, a, b, c, d);
366 fletcher_4_scalar_valid(void)
372 fletcher_4_impl_set(const char *val)
375 uint32_t impl = IMPL_READ(fletcher_4_impl_chosen);
378 val_len = strlen(val);
379 while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */
382 /* check mandatory implementations */
383 for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) {
384 const char *name = fletcher_4_impl_selectors[i].fis_name;
386 if (val_len == strlen(name) &&
387 strncmp(val, name, val_len) == 0) {
388 impl = fletcher_4_impl_selectors[i].fis_sel;
394 if (err != 0 && fletcher_4_initialized) {
395 /* check all supported implementations */
396 for (i = 0; i < fletcher_4_supp_impls_cnt; i++) {
397 const char *name = fletcher_4_supp_impls[i]->name;
399 if (val_len == strlen(name) &&
400 strncmp(val, name, val_len) == 0) {
409 atomic_swap_32(&fletcher_4_impl_chosen, impl);
417 * Returns the Fletcher 4 operations for checksums. When a SIMD
418 * implementation is not allowed in the current context, then fallback
419 * to the fastest generic implementation.
421 static inline const fletcher_4_ops_t *
422 fletcher_4_impl_get(void)
425 return (&fletcher_4_superscalar4_ops);
427 const fletcher_4_ops_t *ops = NULL;
428 uint32_t impl = IMPL_READ(fletcher_4_impl_chosen);
432 ASSERT(fletcher_4_initialized);
433 ops = &fletcher_4_fastest_impl;
436 /* Cycle through supported implementations */
437 ASSERT(fletcher_4_initialized);
438 ASSERT3U(fletcher_4_supp_impls_cnt, >, 0);
439 static uint32_t cycle_count = 0;
440 uint32_t idx = (++cycle_count) % fletcher_4_supp_impls_cnt;
441 ops = fletcher_4_supp_impls[idx];
444 ASSERT3U(fletcher_4_supp_impls_cnt, >, 0);
445 ASSERT3U(impl, <, fletcher_4_supp_impls_cnt);
446 ops = fletcher_4_supp_impls[impl];
450 ASSERT3P(ops, !=, NULL);
456 fletcher_4_native_impl(const void *buf, uint64_t size, zio_cksum_t *zcp)
458 fletcher_4_ctx_t ctx;
459 const fletcher_4_ops_t *ops = fletcher_4_impl_get();
461 ops->init_native(&ctx);
462 ops->compute_native(&ctx, buf, size);
463 ops->fini_native(&ctx, zcp);
467 fletcher_4_native(const void *buf, uint64_t size,
468 const void *ctx_template, zio_cksum_t *zcp)
471 const uint64_t p2size = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE);
473 ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t)));
475 if (size == 0 || p2size == 0) {
476 ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
479 fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp,
482 fletcher_4_native_impl(buf, p2size, zcp);
485 fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp,
486 (char *)buf + p2size, size - p2size);
491 fletcher_4_native_varsize(const void *buf, uint64_t size, zio_cksum_t *zcp)
493 ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
494 fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp, buf, size);
498 fletcher_4_byteswap_impl(const void *buf, uint64_t size, zio_cksum_t *zcp)
500 fletcher_4_ctx_t ctx;
501 const fletcher_4_ops_t *ops = fletcher_4_impl_get();
503 ops->init_byteswap(&ctx);
504 ops->compute_byteswap(&ctx, buf, size);
505 ops->fini_byteswap(&ctx, zcp);
509 fletcher_4_byteswap(const void *buf, uint64_t size,
510 const void *ctx_template, zio_cksum_t *zcp)
513 const uint64_t p2size = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE);
515 ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t)));
517 if (size == 0 || p2size == 0) {
518 ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0);
521 fletcher_4_scalar_byteswap((fletcher_4_ctx_t *)zcp,
524 fletcher_4_byteswap_impl(buf, p2size, zcp);
527 fletcher_4_scalar_byteswap((fletcher_4_ctx_t *)zcp,
528 (char *)buf + p2size, size - p2size);
532 /* Incremental Fletcher 4 */
534 #define ZFS_FLETCHER_4_INC_MAX_SIZE (8ULL << 20)
537 fletcher_4_incremental_combine(zio_cksum_t *zcp, const uint64_t size,
538 const zio_cksum_t *nzcp)
540 const uint64_t c1 = size / sizeof (uint32_t);
541 const uint64_t c2 = c1 * (c1 + 1) / 2;
542 const uint64_t c3 = c2 * (c1 + 2) / 3;
545 * Value of 'c3' overflows on buffer sizes close to 16MiB. For that
546 * reason we split incremental fletcher4 computation of large buffers
547 * to steps of (ZFS_FLETCHER_4_INC_MAX_SIZE) size.
549 ASSERT3U(size, <=, ZFS_FLETCHER_4_INC_MAX_SIZE);
551 zcp->zc_word[3] += nzcp->zc_word[3] + c1 * zcp->zc_word[2] +
552 c2 * zcp->zc_word[1] + c3 * zcp->zc_word[0];
553 zcp->zc_word[2] += nzcp->zc_word[2] + c1 * zcp->zc_word[1] +
554 c2 * zcp->zc_word[0];
555 zcp->zc_word[1] += nzcp->zc_word[1] + c1 * zcp->zc_word[0];
556 zcp->zc_word[0] += nzcp->zc_word[0];
560 fletcher_4_incremental_impl(boolean_t native, const void *buf, uint64_t size,
565 uint64_t len = MIN(size, ZFS_FLETCHER_4_INC_MAX_SIZE);
568 fletcher_4_native(buf, len, NULL, &nzc);
570 fletcher_4_byteswap(buf, len, NULL, &nzc);
572 fletcher_4_incremental_combine(zcp, len, &nzc);
580 fletcher_4_incremental_native(void *buf, size_t size, void *data)
582 zio_cksum_t *zcp = data;
583 /* Use scalar impl to directly update cksum of small blocks */
584 if (size < SPA_MINBLOCKSIZE)
585 fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp, buf, size);
587 fletcher_4_incremental_impl(B_TRUE, buf, size, zcp);
592 fletcher_4_incremental_byteswap(void *buf, size_t size, void *data)
594 zio_cksum_t *zcp = data;
595 /* Use scalar impl to directly update cksum of small blocks */
596 if (size < SPA_MINBLOCKSIZE)
597 fletcher_4_scalar_byteswap((fletcher_4_ctx_t *)zcp, buf, size);
599 fletcher_4_incremental_impl(B_FALSE, buf, size, zcp);
608 fletcher_4_kstat_headers(char *buf, size_t size)
612 off += snprintf(buf + off, size, "%-17s", "implementation");
613 off += snprintf(buf + off, size - off, "%-15s", "native");
614 (void) snprintf(buf + off, size - off, "%-15s\n", "byteswap");
620 fletcher_4_kstat_data(char *buf, size_t size, void *data)
622 struct fletcher_4_kstat *fastest_stat =
623 &fletcher_4_stat_data[fletcher_4_supp_impls_cnt];
624 struct fletcher_4_kstat *curr_stat = (struct fletcher_4_kstat *)data;
627 if (curr_stat == fastest_stat) {
628 off += snprintf(buf + off, size - off, "%-17s", "fastest");
629 off += snprintf(buf + off, size - off, "%-15s",
630 fletcher_4_supp_impls[fastest_stat->native]->name);
631 off += snprintf(buf + off, size - off, "%-15s\n",
632 fletcher_4_supp_impls[fastest_stat->byteswap]->name);
634 ptrdiff_t id = curr_stat - fletcher_4_stat_data;
636 off += snprintf(buf + off, size - off, "%-17s",
637 fletcher_4_supp_impls[id]->name);
638 off += snprintf(buf + off, size - off, "%-15llu",
639 (u_longlong_t)curr_stat->native);
640 off += snprintf(buf + off, size - off, "%-15llu\n",
641 (u_longlong_t)curr_stat->byteswap);
648 fletcher_4_kstat_addr(kstat_t *ksp, loff_t n)
650 if (n <= fletcher_4_supp_impls_cnt)
651 ksp->ks_private = (void *) (fletcher_4_stat_data + n);
653 ksp->ks_private = NULL;
655 return (ksp->ks_private);
659 #define FLETCHER_4_FASTEST_FN_COPY(type, src) \
661 fletcher_4_fastest_impl.init_ ## type = src->init_ ## type; \
662 fletcher_4_fastest_impl.fini_ ## type = src->fini_ ## type; \
663 fletcher_4_fastest_impl.compute_ ## type = src->compute_ ## type; \
666 #define FLETCHER_4_BENCH_NS (MSEC2NSEC(1)) /* 1ms */
668 typedef void fletcher_checksum_func_t(const void *, uint64_t, const void *,
673 fletcher_4_benchmark_impl(boolean_t native, char *data, uint64_t data_size)
676 struct fletcher_4_kstat *fastest_stat =
677 &fletcher_4_stat_data[fletcher_4_supp_impls_cnt];
679 uint64_t run_bw, run_time_ns, best_run = 0;
681 uint32_t i, l, sel_save = IMPL_READ(fletcher_4_impl_chosen);
683 fletcher_checksum_func_t *fletcher_4_test = native ?
684 fletcher_4_native : fletcher_4_byteswap;
686 for (i = 0; i < fletcher_4_supp_impls_cnt; i++) {
687 struct fletcher_4_kstat *stat = &fletcher_4_stat_data[i];
688 uint64_t run_count = 0;
690 /* temporary set an implementation */
691 fletcher_4_impl_chosen = i;
696 for (l = 0; l < 32; l++, run_count++)
697 fletcher_4_test(data, data_size, NULL, &zc);
699 run_time_ns = gethrtime() - start;
700 } while (run_time_ns < FLETCHER_4_BENCH_NS);
703 run_bw = data_size * run_count * NANOSEC;
704 run_bw /= run_time_ns; /* B/s */
707 stat->native = run_bw;
709 stat->byteswap = run_bw;
711 if (run_bw > best_run) {
715 fastest_stat->native = i;
716 FLETCHER_4_FASTEST_FN_COPY(native,
717 fletcher_4_supp_impls[i]);
719 fastest_stat->byteswap = i;
720 FLETCHER_4_FASTEST_FN_COPY(byteswap,
721 fletcher_4_supp_impls[i]);
726 /* restore original selection */
727 atomic_swap_32(&fletcher_4_impl_chosen, sel_save);
732 * Initialize and benchmark all supported implementations.
735 fletcher_4_benchmark(void)
737 fletcher_4_ops_t *curr_impl;
740 /* Move supported implementations into fletcher_4_supp_impls */
741 for (i = 0, c = 0; i < ARRAY_SIZE(fletcher_4_impls); i++) {
742 curr_impl = (fletcher_4_ops_t *)fletcher_4_impls[i];
744 if (curr_impl->valid && curr_impl->valid())
745 fletcher_4_supp_impls[c++] = curr_impl;
747 membar_producer(); /* complete fletcher_4_supp_impls[] init */
748 fletcher_4_supp_impls_cnt = c; /* number of supported impl */
751 static const size_t data_size = 1 << SPA_OLD_MAXBLOCKSHIFT; /* 128kiB */
752 char *databuf = vmem_alloc(data_size, KM_SLEEP);
754 for (i = 0; i < data_size / sizeof (uint64_t); i++)
755 ((uint64_t *)databuf)[i] = (uintptr_t)(databuf+i); /* warm-up */
757 fletcher_4_benchmark_impl(B_FALSE, databuf, data_size);
758 fletcher_4_benchmark_impl(B_TRUE, databuf, data_size);
760 vmem_free(databuf, data_size);
763 * Skip the benchmark in user space to avoid impacting libzpool
764 * consumers (zdb, zhack, zinject, ztest). The last implementation
765 * is assumed to be the fastest and used by default.
767 memcpy(&fletcher_4_fastest_impl,
768 fletcher_4_supp_impls[fletcher_4_supp_impls_cnt - 1],
769 sizeof (fletcher_4_fastest_impl));
770 fletcher_4_fastest_impl.name = "fastest";
776 fletcher_4_init(void)
778 /* Determine the fastest available implementation. */
779 fletcher_4_benchmark();
782 /* Install kstats for all implementations */
783 fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench", "misc",
784 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
785 if (fletcher_4_kstat != NULL) {
786 fletcher_4_kstat->ks_data = NULL;
787 fletcher_4_kstat->ks_ndata = UINT32_MAX;
788 kstat_set_raw_ops(fletcher_4_kstat,
789 fletcher_4_kstat_headers,
790 fletcher_4_kstat_data,
791 fletcher_4_kstat_addr);
792 kstat_install(fletcher_4_kstat);
796 /* Finish initialization */
797 fletcher_4_initialized = B_TRUE;
801 fletcher_4_fini(void)
804 if (fletcher_4_kstat != NULL) {
805 kstat_delete(fletcher_4_kstat);
806 fletcher_4_kstat = NULL;
814 abd_fletcher_4_init(zio_abd_checksum_data_t *cdp)
816 const fletcher_4_ops_t *ops = fletcher_4_impl_get();
817 cdp->acd_private = (void *) ops;
819 if (cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE)
820 ops->init_native(cdp->acd_ctx);
822 ops->init_byteswap(cdp->acd_ctx);
826 abd_fletcher_4_fini(zio_abd_checksum_data_t *cdp)
828 fletcher_4_ops_t *ops = (fletcher_4_ops_t *)cdp->acd_private;
832 if (cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE)
833 ops->fini_native(cdp->acd_ctx, cdp->acd_zcp);
835 ops->fini_byteswap(cdp->acd_ctx, cdp->acd_zcp);
839 abd_fletcher_4_simd2scalar(boolean_t native, void *data, size_t size,
840 zio_abd_checksum_data_t *cdp)
842 zio_cksum_t *zcp = cdp->acd_zcp;
844 ASSERT3U(size, <, FLETCHER_MIN_SIMD_SIZE);
846 abd_fletcher_4_fini(cdp);
847 cdp->acd_private = (void *)&fletcher_4_scalar_ops;
850 fletcher_4_incremental_native(data, size, zcp);
852 fletcher_4_incremental_byteswap(data, size, zcp);
856 abd_fletcher_4_iter(void *data, size_t size, void *private)
858 zio_abd_checksum_data_t *cdp = (zio_abd_checksum_data_t *)private;
859 fletcher_4_ctx_t *ctx = cdp->acd_ctx;
860 fletcher_4_ops_t *ops = (fletcher_4_ops_t *)cdp->acd_private;
861 boolean_t native = cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE;
862 uint64_t asize = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE);
864 ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t)));
868 ops->compute_native(ctx, data, asize);
870 ops->compute_byteswap(ctx, data, asize);
873 data = (char *)data + asize;
877 ASSERT3U(size, <, FLETCHER_MIN_SIMD_SIZE);
878 /* At this point we have to switch to scalar impl */
879 abd_fletcher_4_simd2scalar(native, data, size, cdp);
885 zio_abd_checksum_func_t fletcher_4_abd_ops = {
886 .acf_init = abd_fletcher_4_init,
887 .acf_fini = abd_fletcher_4_fini,
888 .acf_iter = abd_fletcher_4_iter
893 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
895 #if defined(__linux__)
898 fletcher_4_param_get(char *buffer, zfs_kernel_param_t *unused)
900 const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen);
905 fmt = IMPL_FMT(impl, IMPL_FASTEST);
906 cnt += sprintf(buffer + cnt, fmt, "fastest");
908 /* list all supported implementations */
909 for (uint32_t i = 0; i < fletcher_4_supp_impls_cnt; ++i) {
910 fmt = IMPL_FMT(impl, i);
911 cnt += sprintf(buffer + cnt, fmt,
912 fletcher_4_supp_impls[i]->name);
919 fletcher_4_param_set(const char *val, zfs_kernel_param_t *unused)
921 return (fletcher_4_impl_set(val));
926 #include <sys/sbuf.h>
929 fletcher_4_param(ZFS_MODULE_PARAM_ARGS)
933 if (req->newptr == NULL) {
934 const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen);
935 const int init_buflen = 64;
939 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
942 fmt = IMPL_FMT(impl, IMPL_FASTEST);
943 (void) sbuf_printf(s, fmt, "fastest");
945 /* list all supported implementations */
946 for (uint32_t i = 0; i < fletcher_4_supp_impls_cnt; ++i) {
947 fmt = IMPL_FMT(impl, i);
948 (void) sbuf_printf(s, fmt,
949 fletcher_4_supp_impls[i]->name);
952 err = sbuf_finish(s);
960 err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
963 return (-fletcher_4_impl_set(buf));
971 * Choose a fletcher 4 implementation in ZFS.
972 * Users can choose "cycle" to exercise all implementations, but this is
973 * for testing purpose therefore it can only be set in user space.
975 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, fletcher_4_impl,
976 fletcher_4_param_set, fletcher_4_param_get, ZMOD_RW,
977 "Select fletcher 4 implementation.");
979 EXPORT_SYMBOL(fletcher_init);
980 EXPORT_SYMBOL(fletcher_2_incremental_native);
981 EXPORT_SYMBOL(fletcher_2_incremental_byteswap);
982 EXPORT_SYMBOL(fletcher_4_init);
983 EXPORT_SYMBOL(fletcher_4_fini);
984 EXPORT_SYMBOL(fletcher_2_native);
985 EXPORT_SYMBOL(fletcher_2_byteswap);
986 EXPORT_SYMBOL(fletcher_4_native);
987 EXPORT_SYMBOL(fletcher_4_native_varsize);
988 EXPORT_SYMBOL(fletcher_4_byteswap);
989 EXPORT_SYMBOL(fletcher_4_incremental_native);
990 EXPORT_SYMBOL(fletcher_4_incremental_byteswap);
991 EXPORT_SYMBOL(fletcher_4_abd_ops);