sys/contrib/openzfs/module/zfs/vdev_raidz_math.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
  23  */
  24
  25 #include <sys/simd.h>
  26 #include <sys/zfs_context.h>
  27 #include <sys/types.h>
  28 #include <sys/zio.h>
  29 #include <sys/debug.h>
  30 #include <sys/zfs_debug.h>
  31 #include <sys/vdev_raidz.h>
  32 #include <sys/vdev_raidz_impl.h>
  33
  34 /* Opaque implementation with NULL methods to represent original methods */
  35 static const raidz_impl_ops_t vdev_raidz_original_impl = {
  36         .name = "original",
  37         .is_supported = raidz_will_scalar_work,
  38 };
  39
  40 /* RAIDZ parity op that contain the fastest methods */
  41 static raidz_impl_ops_t vdev_raidz_fastest_impl = {
  42         .name = "fastest"
  43 };
  44
  45 /* All compiled in implementations */
  46 static const raidz_impl_ops_t *const raidz_all_maths[] = {
  47         &vdev_raidz_original_impl,
  48         &vdev_raidz_scalar_impl,
  49 #if defined(__x86_64) && defined(HAVE_SSE2)     /* only x86_64 for now */
  50         &vdev_raidz_sse2_impl,
  51 #endif
  52 #if defined(__x86_64) && defined(HAVE_SSSE3)    /* only x86_64 for now */
  53         &vdev_raidz_ssse3_impl,
  54 #endif
  55 #if defined(__x86_64) && defined(HAVE_AVX2)     /* only x86_64 for now */
  56         &vdev_raidz_avx2_impl,
  57 #endif
  58 #if defined(__x86_64) && defined(HAVE_AVX512F)  /* only x86_64 for now */
  59         &vdev_raidz_avx512f_impl,
  60 #endif
  61 #if defined(__x86_64) && defined(HAVE_AVX512BW) /* only x86_64 for now */
  62         &vdev_raidz_avx512bw_impl,
  63 #endif
  64 #if defined(__aarch64__) && !defined(__FreeBSD__)
  65         &vdev_raidz_aarch64_neon_impl,
  66         &vdev_raidz_aarch64_neonx2_impl,
  67 #endif
  68 #if defined(__powerpc__) && defined(__altivec__)
  69         &vdev_raidz_powerpc_altivec_impl,
  70 #endif
  71 };
  72
  73 /* Indicate that benchmark has been completed */
  74 static boolean_t raidz_math_initialized = B_FALSE;
  75
  76 /* Select raidz implementation */
  77 #define IMPL_FASTEST    (UINT32_MAX)
  78 #define IMPL_CYCLE      (UINT32_MAX - 1)
  79 #define IMPL_ORIGINAL   (0)
  80 #define IMPL_SCALAR     (1)
  81
  82 #define RAIDZ_IMPL_READ(i)      (*(volatile uint32_t *) &(i))
  83
  84 static uint32_t zfs_vdev_raidz_impl = IMPL_SCALAR;
  85 static uint32_t user_sel_impl = IMPL_FASTEST;
  86
  87 /* Hold all supported implementations */
  88 static size_t raidz_supp_impl_cnt = 0;
  89 static raidz_impl_ops_t *raidz_supp_impl[ARRAY_SIZE(raidz_all_maths)];
  90
  91 #if defined(_KERNEL)
  92 /*
  93  * kstats values for supported implementations
  94  * Values represent per disk throughput of 8 disk+parity raidz vdev [B/s]
  95  */
  96 static raidz_impl_kstat_t raidz_impl_kstats[ARRAY_SIZE(raidz_all_maths) + 1];
  97
  98 /* kstat for benchmarked implementations */
  99 static kstat_t *raidz_math_kstat = NULL;
 100 #endif
 101
 102 /*
 103  * Returns the RAIDZ operations for raidz_map() parity calculations.   When
 104  * a SIMD implementation is not allowed in the current context, then fallback
 105  * to the fastest generic implementation.
 106  */
 107 const raidz_impl_ops_t *
 108 vdev_raidz_math_get_ops(void)
 109 {
 110         if (!kfpu_allowed())
 111                 return (&vdev_raidz_scalar_impl);
 112
 113         raidz_impl_ops_t *ops = NULL;
 114         const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl);
 115
 116         switch (impl) {
 117         case IMPL_FASTEST:
 118                 ASSERT(raidz_math_initialized);
 119                 ops = &vdev_raidz_fastest_impl;
 120                 break;
 121         case IMPL_CYCLE:
 122                 /* Cycle through all supported implementations */
 123                 ASSERT(raidz_math_initialized);
 124                 ASSERT3U(raidz_supp_impl_cnt, >, 0);
 125                 static size_t cycle_impl_idx = 0;
 126                 size_t idx = (++cycle_impl_idx) % raidz_supp_impl_cnt;
 127                 ops = raidz_supp_impl[idx];
 128                 break;
 129         case IMPL_ORIGINAL:
 130                 ops = (raidz_impl_ops_t *)&vdev_raidz_original_impl;
 131                 break;
 132         case IMPL_SCALAR:
 133                 ops = (raidz_impl_ops_t *)&vdev_raidz_scalar_impl;
 134                 break;
 135         default:
 136                 ASSERT3U(impl, <, raidz_supp_impl_cnt);
 137                 ASSERT3U(raidz_supp_impl_cnt, >, 0);
 138                 if (impl < ARRAY_SIZE(raidz_all_maths))
 139                         ops = raidz_supp_impl[impl];
 140                 break;
 141         }
 142
 143         ASSERT3P(ops, !=, NULL);
 144
 145         return (ops);
 146 }
 147
 148 /*
 149  * Select parity generation method for raidz_map
 150  */
 151 int
 152 vdev_raidz_math_generate(raidz_map_t *rm, raidz_row_t *rr)
 153 {
 154         raidz_gen_f gen_parity = NULL;
 155
 156         switch (raidz_parity(rm)) {
 157                 case 1:
 158                         gen_parity = rm->rm_ops->gen[RAIDZ_GEN_P];
 159                         break;
 160                 case 2:
 161                         gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQ];
 162                         break;
 163                 case 3:
 164                         gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQR];
 165                         break;
 166                 default:
 167                         gen_parity = NULL;
 168                         cmn_err(CE_PANIC, "invalid RAID-Z configuration %llu",
 169                             (u_longlong_t)raidz_parity(rm));
 170                         break;
 171         }
 172
 173         /* if method is NULL execute the original implementation */
 174         if (gen_parity == NULL)
 175                 return (RAIDZ_ORIGINAL_IMPL);
 176
 177         gen_parity(rr);
 178
 179         return (0);
 180 }
 181
 182 static raidz_rec_f
 183 reconstruct_fun_p_sel(raidz_map_t *rm, const int *parity_valid,
 184     const int nbaddata)
 185 {
 186         if (nbaddata == 1 && parity_valid[CODE_P]) {
 187                 return (rm->rm_ops->rec[RAIDZ_REC_P]);
 188         }
 189         return ((raidz_rec_f) NULL);
 190 }
 191
 192 static raidz_rec_f
 193 reconstruct_fun_pq_sel(raidz_map_t *rm, const int *parity_valid,
 194     const int nbaddata)
 195 {
 196         if (nbaddata == 1) {
 197                 if (parity_valid[CODE_P]) {
 198                         return (rm->rm_ops->rec[RAIDZ_REC_P]);
 199                 } else if (parity_valid[CODE_Q]) {
 200                         return (rm->rm_ops->rec[RAIDZ_REC_Q]);
 201                 }
 202         } else if (nbaddata == 2 &&
 203             parity_valid[CODE_P] && parity_valid[CODE_Q]) {
 204                 return (rm->rm_ops->rec[RAIDZ_REC_PQ]);
 205         }
 206         return ((raidz_rec_f) NULL);
 207 }
 208
 209 static raidz_rec_f
 210 reconstruct_fun_pqr_sel(raidz_map_t *rm, const int *parity_valid,
 211     const int nbaddata)
 212 {
 213         if (nbaddata == 1) {
 214                 if (parity_valid[CODE_P]) {
 215                         return (rm->rm_ops->rec[RAIDZ_REC_P]);
 216                 } else if (parity_valid[CODE_Q]) {
 217                         return (rm->rm_ops->rec[RAIDZ_REC_Q]);
 218                 } else if (parity_valid[CODE_R]) {
 219                         return (rm->rm_ops->rec[RAIDZ_REC_R]);
 220                 }
 221         } else if (nbaddata == 2) {
 222                 if (parity_valid[CODE_P] && parity_valid[CODE_Q]) {
 223                         return (rm->rm_ops->rec[RAIDZ_REC_PQ]);
 224                 } else if (parity_valid[CODE_P] && parity_valid[CODE_R]) {
 225                         return (rm->rm_ops->rec[RAIDZ_REC_PR]);
 226                 } else if (parity_valid[CODE_Q] && parity_valid[CODE_R]) {
 227                         return (rm->rm_ops->rec[RAIDZ_REC_QR]);
 228                 }
 229         } else if (nbaddata == 3 &&
 230             parity_valid[CODE_P] && parity_valid[CODE_Q] &&
 231             parity_valid[CODE_R]) {
 232                 return (rm->rm_ops->rec[RAIDZ_REC_PQR]);
 233         }
 234         return ((raidz_rec_f) NULL);
 235 }
 236
 237 /*
 238  * Select data reconstruction method for raidz_map
 239  * @parity_valid - Parity validity flag
 240  * @dt           - Failed data index array
 241  * @nbaddata     - Number of failed data columns
 242  */
 243 int
 244 vdev_raidz_math_reconstruct(raidz_map_t *rm, raidz_row_t *rr,
 245     const int *parity_valid, const int *dt, const int nbaddata)
 246 {
 247         raidz_rec_f rec_fn = NULL;
 248
 249         switch (raidz_parity(rm)) {
 250         case PARITY_P:
 251                 rec_fn = reconstruct_fun_p_sel(rm, parity_valid, nbaddata);
 252                 break;
 253         case PARITY_PQ:
 254                 rec_fn = reconstruct_fun_pq_sel(rm, parity_valid, nbaddata);
 255                 break;
 256         case PARITY_PQR:
 257                 rec_fn = reconstruct_fun_pqr_sel(rm, parity_valid, nbaddata);
 258                 break;
 259         default:
 260                 cmn_err(CE_PANIC, "invalid RAID-Z configuration %llu",
 261                     (u_longlong_t)raidz_parity(rm));
 262                 break;
 263         }
 264
 265         if (rec_fn == NULL)
 266                 return (RAIDZ_ORIGINAL_IMPL);
 267         else
 268                 return (rec_fn(rr, dt));
 269 }
 270
 271 const char *const raidz_gen_name[] = {
 272         "gen_p", "gen_pq", "gen_pqr"
 273 };
 274 const char *const raidz_rec_name[] = {
 275         "rec_p", "rec_q", "rec_r",
 276         "rec_pq", "rec_pr", "rec_qr", "rec_pqr"
 277 };
 278
 279 #if defined(_KERNEL)
 280
 281 #define RAIDZ_KSTAT_LINE_LEN    (17 + 10*12 + 1)
 282
 283 static int
 284 raidz_math_kstat_headers(char *buf, size_t size)
 285 {
 286         ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN);
 287
 288         ssize_t off = kmem_scnprintf(buf, size, "%-17s", "implementation");
 289
 290         for (int i = 0; i < ARRAY_SIZE(raidz_gen_name); i++)
 291                 off += kmem_scnprintf(buf + off, size - off, "%-16s",
 292                     raidz_gen_name[i]);
 293
 294         for (int i = 0; i < ARRAY_SIZE(raidz_rec_name); i++)
 295                 off += kmem_scnprintf(buf + off, size - off, "%-16s",
 296                     raidz_rec_name[i]);
 297
 298         (void) kmem_scnprintf(buf + off, size - off, "\n");
 299
 300         return (0);
 301 }
 302
 303 static int
 304 raidz_math_kstat_data(char *buf, size_t size, void *data)
 305 {
 306         raidz_impl_kstat_t *fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
 307         raidz_impl_kstat_t *cstat = (raidz_impl_kstat_t *)data;
 308         ssize_t off = 0;
 309         int i;
 310
 311         ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN);
 312
 313         if (cstat == fstat) {
 314                 off += kmem_scnprintf(buf + off, size - off, "%-17s",
 315                     "fastest");
 316
 317                 for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) {
 318                         int id = fstat->gen[i];
 319                         off += kmem_scnprintf(buf + off, size - off, "%-16s",
 320                             raidz_supp_impl[id]->name);
 321                 }
 322                 for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) {
 323                         int id = fstat->rec[i];
 324                         off += kmem_scnprintf(buf + off, size - off, "%-16s",
 325                             raidz_supp_impl[id]->name);
 326                 }
 327         } else {
 328                 ptrdiff_t id = cstat - raidz_impl_kstats;
 329
 330                 off += kmem_scnprintf(buf + off, size - off, "%-17s",
 331                     raidz_supp_impl[id]->name);
 332
 333                 for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++)
 334                         off += kmem_scnprintf(buf + off, size - off, "%-16llu",
 335                             (u_longlong_t)cstat->gen[i]);
 336
 337                 for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++)
 338                         off += kmem_scnprintf(buf + off, size - off, "%-16llu",
 339                             (u_longlong_t)cstat->rec[i]);
 340         }
 341
 342         (void) kmem_scnprintf(buf + off, size - off, "\n");
 343
 344         return (0);
 345 }
 346
 347 static void *
 348 raidz_math_kstat_addr(kstat_t *ksp, loff_t n)
 349 {
 350         if (n <= raidz_supp_impl_cnt)
 351                 ksp->ks_private = (void *) (raidz_impl_kstats + n);
 352         else
 353                 ksp->ks_private = NULL;
 354
 355         return (ksp->ks_private);
 356 }
 357
 358 #define BENCH_D_COLS    (8ULL)
 359 #define BENCH_COLS      (BENCH_D_COLS + PARITY_PQR)
 360 #define BENCH_ZIO_SIZE  (1ULL << SPA_OLD_MAXBLOCKSHIFT) /* 128 kiB */
 361 #define BENCH_NS        MSEC2NSEC(1)                    /* 1ms */
 362
 363 typedef void (*benchmark_fn)(raidz_map_t *rm, const int fn);
 364
 365 static void
 366 benchmark_gen_impl(raidz_map_t *rm, const int fn)
 367 {
 368         (void) fn;
 369         vdev_raidz_generate_parity(rm);
 370 }
 371
 372 static void
 373 benchmark_rec_impl(raidz_map_t *rm, const int fn)
 374 {
 375         static const int rec_tgt[7][3] = {
 376                 {1, 2, 3},      /* rec_p:   bad QR & D[0]       */
 377                 {0, 2, 3},      /* rec_q:   bad PR & D[0]       */
 378                 {0, 1, 3},      /* rec_r:   bad PQ & D[0]       */
 379                 {2, 3, 4},      /* rec_pq:  bad R  & D[0][1]    */
 380                 {1, 3, 4},      /* rec_pr:  bad Q  & D[0][1]    */
 381                 {0, 3, 4},      /* rec_qr:  bad P  & D[0][1]    */
 382                 {3, 4, 5}       /* rec_pqr: bad    & D[0][1][2] */
 383         };
 384
 385         vdev_raidz_reconstruct(rm, rec_tgt[fn], 3);
 386 }
 387
 388 /*
 389  * Benchmarking of all supported implementations (raidz_supp_impl_cnt)
 390  * is performed by setting the rm_ops pointer and calling the top level
 391  * generate/reconstruct methods of bench_rm.
 392  */
 393 static void
 394 benchmark_raidz_impl(raidz_map_t *bench_rm, const int fn, benchmark_fn bench_fn)
 395 {
 396         uint64_t run_cnt, speed, best_speed = 0;
 397         hrtime_t t_start, t_diff;
 398         raidz_impl_ops_t *curr_impl;
 399         raidz_impl_kstat_t *fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
 400         int impl, i;
 401
 402         for (impl = 0; impl < raidz_supp_impl_cnt; impl++) {
 403                 /* set an implementation to benchmark */
 404                 curr_impl = raidz_supp_impl[impl];
 405                 bench_rm->rm_ops = curr_impl;
 406
 407                 run_cnt = 0;
 408                 t_start = gethrtime();
 409
 410                 do {
 411                         for (i = 0; i < 5; i++, run_cnt++)
 412                                 bench_fn(bench_rm, fn);
 413
 414                         t_diff = gethrtime() - t_start;
 415                 } while (t_diff < BENCH_NS);
 416
 417                 speed = run_cnt * BENCH_ZIO_SIZE * NANOSEC;
 418                 speed /= (t_diff * BENCH_COLS);
 419
 420                 if (bench_fn == benchmark_gen_impl)
 421                         raidz_impl_kstats[impl].gen[fn] = speed;
 422                 else
 423                         raidz_impl_kstats[impl].rec[fn] = speed;
 424
 425                 /* Update fastest implementation method */
 426                 if (speed > best_speed) {
 427                         best_speed = speed;
 428
 429                         if (bench_fn == benchmark_gen_impl) {
 430                                 fstat->gen[fn] = impl;
 431                                 vdev_raidz_fastest_impl.gen[fn] =
 432                                     curr_impl->gen[fn];
 433                         } else {
 434                                 fstat->rec[fn] = impl;
 435                                 vdev_raidz_fastest_impl.rec[fn] =
 436                                     curr_impl->rec[fn];
 437                         }
 438                 }
 439         }
 440 }
 441 #endif
 442
 443 /*
 444  * Initialize and benchmark all supported implementations.
 445  */
 446 static void
 447 benchmark_raidz(void)
 448 {
 449         raidz_impl_ops_t *curr_impl;
 450         int i, c;
 451
 452         /* Move supported impl into raidz_supp_impl */
 453         for (i = 0, c = 0; i < ARRAY_SIZE(raidz_all_maths); i++) {
 454                 curr_impl = (raidz_impl_ops_t *)raidz_all_maths[i];
 455
 456                 if (curr_impl->init)
 457                         curr_impl->init();
 458
 459                 if (curr_impl->is_supported())
 460                         raidz_supp_impl[c++] = (raidz_impl_ops_t *)curr_impl;
 461         }
 462         membar_producer();              /* complete raidz_supp_impl[] init */
 463         raidz_supp_impl_cnt = c;        /* number of supported impl */
 464
 465 #if defined(_KERNEL)
 466         abd_t *pabd;
 467         zio_t *bench_zio = NULL;
 468         raidz_map_t *bench_rm = NULL;
 469         uint64_t bench_parity;
 470
 471         /* Fake a zio and run the benchmark on a warmed up buffer */
 472         bench_zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP);
 473         bench_zio->io_offset = 0;
 474         bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */
 475         bench_zio->io_abd = abd_alloc_linear(BENCH_ZIO_SIZE, B_TRUE);
 476         memset(abd_to_buf(bench_zio->io_abd), 0xAA, BENCH_ZIO_SIZE);
 477
 478         /* Benchmark parity generation methods */
 479         for (int fn = 0; fn < RAIDZ_GEN_NUM; fn++) {
 480                 bench_parity = fn + 1;
 481                 /* New raidz_map is needed for each generate_p/q/r */
 482                 bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT,
 483                     BENCH_D_COLS + bench_parity, bench_parity);
 484
 485                 benchmark_raidz_impl(bench_rm, fn, benchmark_gen_impl);
 486
 487                 vdev_raidz_map_free(bench_rm);
 488         }
 489
 490         /* Benchmark data reconstruction methods */
 491         bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT,
 492             BENCH_COLS, PARITY_PQR);
 493
 494         /* Ensure that fake parity blocks are initialized */
 495         for (c = 0; c < bench_rm->rm_row[0]->rr_firstdatacol; c++) {
 496                 pabd = bench_rm->rm_row[0]->rr_col[c].rc_abd;
 497                 memset(abd_to_buf(pabd), 0xAA, abd_get_size(pabd));
 498         }
 499
 500         for (int fn = 0; fn < RAIDZ_REC_NUM; fn++)
 501                 benchmark_raidz_impl(bench_rm, fn, benchmark_rec_impl);
 502
 503         vdev_raidz_map_free(bench_rm);
 504
 505         /* cleanup the bench zio */
 506         abd_free(bench_zio->io_abd);
 507         kmem_free(bench_zio, sizeof (zio_t));
 508 #else
 509         /*
 510          * Skip the benchmark in user space to avoid impacting libzpool
 511          * consumers (zdb, zhack, zinject, ztest).  The last implementation
 512          * is assumed to be the fastest and used by default.
 513          */
 514         memcpy(&vdev_raidz_fastest_impl,
 515             raidz_supp_impl[raidz_supp_impl_cnt - 1],
 516             sizeof (vdev_raidz_fastest_impl));
 517         strcpy(vdev_raidz_fastest_impl.name, "fastest");
 518 #endif /* _KERNEL */
 519 }
 520
 521 void
 522 vdev_raidz_math_init(void)
 523 {
 524         /* Determine the fastest available implementation. */
 525         benchmark_raidz();
 526
 527 #if defined(_KERNEL)
 528         /* Install kstats for all implementations */
 529         raidz_math_kstat = kstat_create("zfs", 0, "vdev_raidz_bench", "misc",
 530             KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
 531         if (raidz_math_kstat != NULL) {
 532                 raidz_math_kstat->ks_data = NULL;
 533                 raidz_math_kstat->ks_ndata = UINT32_MAX;
 534                 kstat_set_raw_ops(raidz_math_kstat,
 535                     raidz_math_kstat_headers,
 536                     raidz_math_kstat_data,
 537                     raidz_math_kstat_addr);
 538                 kstat_install(raidz_math_kstat);
 539         }
 540 #endif
 541
 542         /* Finish initialization */
 543         atomic_swap_32(&zfs_vdev_raidz_impl, user_sel_impl);
 544         raidz_math_initialized = B_TRUE;
 545 }
 546
 547 void
 548 vdev_raidz_math_fini(void)
 549 {
 550         raidz_impl_ops_t const *curr_impl;
 551
 552 #if defined(_KERNEL)
 553         if (raidz_math_kstat != NULL) {
 554                 kstat_delete(raidz_math_kstat);
 555                 raidz_math_kstat = NULL;
 556         }
 557 #endif
 558
 559         for (int i = 0; i < ARRAY_SIZE(raidz_all_maths); i++) {
 560                 curr_impl = raidz_all_maths[i];
 561                 if (curr_impl->fini)
 562                         curr_impl->fini();
 563         }
 564 }
 565
 566 static const struct {
 567         const char *name;
 568         uint32_t sel;
 569 } math_impl_opts[] = {
 570                 { "cycle",      IMPL_CYCLE },
 571                 { "fastest",    IMPL_FASTEST },
 572                 { "original",   IMPL_ORIGINAL },
 573                 { "scalar",     IMPL_SCALAR }
 574 };
 575
 576 /*
 577  * Function sets desired raidz implementation.
 578  *
 579  * If we are called before init(), user preference will be saved in
 580  * user_sel_impl, and applied in later init() call. This occurs when module
 581  * parameter is specified on module load. Otherwise, directly update
 582  * zfs_vdev_raidz_impl.
 583  *
 584  * @val         Name of raidz implementation to use
 585  * @param       Unused.
 586  */
 587 int
 588 vdev_raidz_impl_set(const char *val)
 589 {
 590         int err = -EINVAL;
 591         char req_name[RAIDZ_IMPL_NAME_MAX];
 592         uint32_t impl = RAIDZ_IMPL_READ(user_sel_impl);
 593         size_t i;
 594
 595         /* sanitize input */
 596         i = strnlen(val, RAIDZ_IMPL_NAME_MAX);
 597         if (i == 0 || i == RAIDZ_IMPL_NAME_MAX)
 598                 return (err);
 599
 600         strlcpy(req_name, val, RAIDZ_IMPL_NAME_MAX);
 601         while (i > 0 && !!isspace(req_name[i-1]))
 602                 i--;
 603         req_name[i] = '\0';
 604
 605         /* Check mandatory options */
 606         for (i = 0; i < ARRAY_SIZE(math_impl_opts); i++) {
 607                 if (strcmp(req_name, math_impl_opts[i].name) == 0) {
 608                         impl = math_impl_opts[i].sel;
 609                         err = 0;
 610                         break;
 611                 }
 612         }
 613
 614         /* check all supported impl if init() was already called */
 615         if (err != 0 && raidz_math_initialized) {
 616                 /* check all supported implementations */
 617                 for (i = 0; i < raidz_supp_impl_cnt; i++) {
 618                         if (strcmp(req_name, raidz_supp_impl[i]->name) == 0) {
 619                                 impl = i;
 620                                 err = 0;
 621                                 break;
 622                         }
 623                 }
 624         }
 625
 626         if (err == 0) {
 627                 if (raidz_math_initialized)
 628                         atomic_swap_32(&zfs_vdev_raidz_impl, impl);
 629                 else
 630                         atomic_swap_32(&user_sel_impl, impl);
 631         }
 632
 633         return (err);
 634 }
 635
 636 #if defined(_KERNEL) && defined(__linux__)
 637
 638 static int
 639 zfs_vdev_raidz_impl_set(const char *val, zfs_kernel_param_t *kp)
 640 {
 641         return (vdev_raidz_impl_set(val));
 642 }
 643
 644 static int
 645 zfs_vdev_raidz_impl_get(char *buffer, zfs_kernel_param_t *kp)
 646 {
 647         int i, cnt = 0;
 648         char *fmt;
 649         const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl);
 650
 651         ASSERT(raidz_math_initialized);
 652
 653         /* list mandatory options */
 654         for (i = 0; i < ARRAY_SIZE(math_impl_opts) - 2; i++) {
 655                 fmt = (impl == math_impl_opts[i].sel) ? "[%s] " : "%s ";
 656                 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt,
 657                     math_impl_opts[i].name);
 658         }
 659
 660         /* list all supported implementations */
 661         for (i = 0; i < raidz_supp_impl_cnt; i++) {
 662                 fmt = (i == impl) ? "[%s] " : "%s ";
 663                 cnt += kmem_scnprintf(buffer + cnt, PAGE_SIZE - cnt, fmt,
 664                     raidz_supp_impl[i]->name);
 665         }
 666
 667         return (cnt);
 668 }
 669
 670 module_param_call(zfs_vdev_raidz_impl, zfs_vdev_raidz_impl_set,
 671     zfs_vdev_raidz_impl_get, NULL, 0644);
 672 MODULE_PARM_DESC(zfs_vdev_raidz_impl, "Select raidz implementation.");
 673 #endif