module/zfs/vdev_raidz_math.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
  23  */
  24
  25 #include <sys/zfs_context.h>
  26 #include <sys/types.h>
  27 #include <sys/zio.h>
  28 #include <sys/debug.h>
  29 #include <sys/zfs_debug.h>
  30
  31 #include <sys/vdev_raidz.h>
  32 #include <sys/vdev_raidz_impl.h>
  33
  34 extern boolean_t raidz_will_scalar_work(void);
  35
  36 /* Opaque implementation with NULL methods to represent original methods */
  37 static const raidz_impl_ops_t vdev_raidz_original_impl = {
  38         .name = "original",
  39         .is_supported = raidz_will_scalar_work,
  40 };
  41
  42 /* RAIDZ parity op that contain the fastest methods */
  43 static raidz_impl_ops_t vdev_raidz_fastest_impl = {
  44         .name = "fastest"
  45 };
  46
  47 /* ABD BRINGUP -- not ready yet */
  48 #if 1
  49 #ifdef HAVE_SSSE3
  50 #undef HAVE_SSSE3
  51 #endif
  52 #ifdef HAVE_AVX2
  53 #undef HAVE_AVX2
  54 #endif
  55 #endif
  56
  57 /* All compiled in implementations */
  58 const raidz_impl_ops_t *raidz_all_maths[] = {
  59         &vdev_raidz_original_impl,
  60         &vdev_raidz_scalar_impl,
  61 #if defined(__x86_64) && defined(HAVE_SSE2)     /* only x86_64 for now */
  62         &vdev_raidz_sse2_impl,
  63 #endif
  64 #if defined(__x86_64) && defined(HAVE_SSSE3)    /* only x86_64 for now */
  65         &vdev_raidz_ssse3_impl,
  66 #endif
  67 #if defined(__x86_64) && defined(HAVE_AVX2)     /* only x86_64 for now */
  68         &vdev_raidz_avx2_impl,
  69 #endif
  70 #if defined(__x86_64) && defined(HAVE_AVX512F)  /* only x86_64 for now */
  71         &vdev_raidz_avx512f_impl,
  72 #endif
  73 #if defined(__x86_64) && defined(HAVE_AVX512BW) /* only x86_64 for now */
  74         &vdev_raidz_avx512bw_impl,
  75 #endif
  76 #if defined(__aarch64__)
  77         &vdev_raidz_aarch64_neon_impl,
  78         &vdev_raidz_aarch64_neonx2_impl,
  79 #endif
  80 };
  81
  82 /* Indicate that benchmark has been completed */
  83 static boolean_t raidz_math_initialized = B_FALSE;
  84
  85 /* Select raidz implementation */
  86 #define IMPL_FASTEST    (UINT32_MAX)
  87 #define IMPL_CYCLE      (UINT32_MAX - 1)
  88 #define IMPL_ORIGINAL   (0)
  89 #define IMPL_SCALAR     (1)
  90
  91 #define RAIDZ_IMPL_READ(i)      (*(volatile uint32_t *) &(i))
  92
  93 static uint32_t zfs_vdev_raidz_impl = IMPL_SCALAR;
  94 static uint32_t user_sel_impl = IMPL_FASTEST;
  95
  96 /* Hold all supported implementations */
  97 static size_t raidz_supp_impl_cnt = 0;
  98 static raidz_impl_ops_t *raidz_supp_impl[ARRAY_SIZE(raidz_all_maths)];
  99
 100 /*
 101  * kstats values for supported implementations
 102  * Values represent per disk throughput of 8 disk+parity raidz vdev [B/s]
 103  */
 104 static raidz_impl_kstat_t raidz_impl_kstats[ARRAY_SIZE(raidz_all_maths) + 1];
 105
 106 /* kstat for benchmarked implementations */
 107 static kstat_t *raidz_math_kstat = NULL;
 108
 109 /*
 110  * Selects the raidz operation for raidz_map
 111  * If rm_ops is set to NULL original raidz implementation will be used
 112  */
 113 raidz_impl_ops_t *
 114 vdev_raidz_math_get_ops()
 115 {
 116         raidz_impl_ops_t *ops = NULL;
 117         const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl);
 118
 119         switch (impl) {
 120         case IMPL_FASTEST:
 121                 ASSERT(raidz_math_initialized);
 122                 ops = &vdev_raidz_fastest_impl;
 123                 break;
 124 #if !defined(_KERNEL)
 125         case IMPL_CYCLE:
 126         {
 127                 ASSERT(raidz_math_initialized);
 128                 ASSERT3U(raidz_supp_impl_cnt, >, 0);
 129                 /* Cycle through all supported implementations */
 130                 static size_t cycle_impl_idx = 0;
 131                 size_t idx = (++cycle_impl_idx) % raidz_supp_impl_cnt;
 132                 ops = raidz_supp_impl[idx];
 133         }
 134         break;
 135 #endif
 136         case IMPL_ORIGINAL:
 137                 ops = (raidz_impl_ops_t *) &vdev_raidz_original_impl;
 138                 break;
 139         case IMPL_SCALAR:
 140                 ops = (raidz_impl_ops_t *) &vdev_raidz_scalar_impl;
 141                 break;
 142         default:
 143                 ASSERT3U(impl, <, raidz_supp_impl_cnt);
 144                 ASSERT3U(raidz_supp_impl_cnt, >, 0);
 145                 ops = raidz_supp_impl[impl];
 146                 break;
 147         }
 148
 149         ASSERT3P(ops, !=, NULL);
 150
 151         return (ops);
 152 }
 153
 154 /*
 155  * Select parity generation method for raidz_map
 156  */
 157 int
 158 vdev_raidz_math_generate(raidz_map_t *rm)
 159 {
 160         raidz_gen_f gen_parity = NULL;
 161
 162 /* ABD Bringup -- vector code not ready */
 163 #if 0
 164         switch (raidz_parity(rm)) {
 165                 case 1:
 166                         gen_parity = rm->rm_ops->gen[RAIDZ_GEN_P];
 167                         break;
 168                 case 2:
 169                         gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQ];
 170                         break;
 171                 case 3:
 172                         gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQR];
 173                         break;
 174                 default:
 175                         gen_parity = NULL;
 176                         cmn_err(CE_PANIC, "invalid RAID-Z configuration %d",
 177                                 raidz_parity(rm));
 178                         break;
 179         }
 180 #endif
 181
 182         /* if method is NULL execute the original implementation */
 183         if (gen_parity == NULL)
 184                 return (RAIDZ_ORIGINAL_IMPL);
 185
 186         gen_parity(rm);
 187
 188         return (0);
 189 }
 190
 191 /* ABD Bringup -- vector code not ready */
 192 #if 0
 193 static raidz_rec_f
 194 reconstruct_fun_p_sel(raidz_map_t *rm, const int *parity_valid,
 195         const int nbaddata)
 196 {
 197         if (nbaddata == 1 && parity_valid[CODE_P]) {
 198                 return (rm->rm_ops->rec[RAIDZ_REC_P]);
 199         }
 200         return ((raidz_rec_f) NULL);
 201 }
 202
 203 static raidz_rec_f
 204 reconstruct_fun_pq_sel(raidz_map_t *rm, const int *parity_valid,
 205         const int nbaddata)
 206 {
 207         if (nbaddata == 1) {
 208                 if (parity_valid[CODE_P]) {
 209                         return (rm->rm_ops->rec[RAIDZ_REC_P]);
 210                 } else if (parity_valid[CODE_Q]) {
 211                         return (rm->rm_ops->rec[RAIDZ_REC_Q]);
 212                 }
 213         } else if (nbaddata == 2 &&
 214                 parity_valid[CODE_P] && parity_valid[CODE_Q]) {
 215                 return (rm->rm_ops->rec[RAIDZ_REC_PQ]);
 216         }
 217         return ((raidz_rec_f) NULL);
 218 }
 219
 220 static raidz_rec_f
 221 reconstruct_fun_pqr_sel(raidz_map_t *rm, const int *parity_valid,
 222         const int nbaddata)
 223 {
 224         if (nbaddata == 1) {
 225                 if (parity_valid[CODE_P]) {
 226                         return (rm->rm_ops->rec[RAIDZ_REC_P]);
 227                 } else if (parity_valid[CODE_Q]) {
 228                         return (rm->rm_ops->rec[RAIDZ_REC_Q]);
 229                 } else if (parity_valid[CODE_R]) {
 230                         return (rm->rm_ops->rec[RAIDZ_REC_R]);
 231                 }
 232         } else if (nbaddata == 2) {
 233                 if (parity_valid[CODE_P] && parity_valid[CODE_Q]) {
 234                         return (rm->rm_ops->rec[RAIDZ_REC_PQ]);
 235                 } else if (parity_valid[CODE_P] && parity_valid[CODE_R]) {
 236                         return (rm->rm_ops->rec[RAIDZ_REC_PR]);
 237                 } else if (parity_valid[CODE_Q] && parity_valid[CODE_R]) {
 238                         return (rm->rm_ops->rec[RAIDZ_REC_QR]);
 239                 }
 240         } else if (nbaddata == 3 &&
 241                 parity_valid[CODE_P] && parity_valid[CODE_Q] &&
 242                 parity_valid[CODE_R]) {
 243                 return (rm->rm_ops->rec[RAIDZ_REC_PQR]);
 244         }
 245         return ((raidz_rec_f) NULL);
 246 }
 247 #endif
 248
 249 /*
 250  * Select data reconstruction method for raidz_map
 251  * @parity_valid - Parity validity flag
 252  * @dt           - Failed data index array
 253  * @nbaddata     - Number of failed data columns
 254  */
 255 int
 256 vdev_raidz_math_reconstruct(raidz_map_t *rm, const int *parity_valid,
 257         const int *dt, const int nbaddata)
 258 {
 259         raidz_rec_f rec_data = NULL;
 260
 261 /* ABD Bringup -- vector code not ready */
 262 #if 0
 263         switch (raidz_parity(rm)) {
 264         case PARITY_P:
 265                 rec_data = reconstruct_fun_p_sel(rm, parity_valid, nbaddata);
 266                 break;
 267         case PARITY_PQ:
 268                 rec_data = reconstruct_fun_pq_sel(rm, parity_valid, nbaddata);
 269                 break;
 270         case PARITY_PQR:
 271                 rec_data = reconstruct_fun_pqr_sel(rm, parity_valid, nbaddata);
 272                 break;
 273         default:
 274                 cmn_err(CE_PANIC, "invalid RAID-Z configuration %d",
 275                     raidz_parity(rm));
 276                 break;
 277         }
 278 #endif
 279
 280         if (rec_data == NULL)
 281                 return (RAIDZ_ORIGINAL_IMPL);
 282         else
 283                 return (rec_data(rm, dt));
 284 }
 285
 286 const char *raidz_gen_name[] = {
 287         "gen_p", "gen_pq", "gen_pqr"
 288 };
 289 const char *raidz_rec_name[] = {
 290         "rec_p", "rec_q", "rec_r",
 291         "rec_pq", "rec_pr", "rec_qr", "rec_pqr"
 292 };
 293
 294 #define RAIDZ_KSTAT_LINE_LEN    (17 + 10*12 + 1)
 295
 296 static int
 297 raidz_math_kstat_headers(char *buf, size_t size)
 298 {
 299         int i;
 300         ssize_t off;
 301
 302         ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN);
 303
 304         off = snprintf(buf, size, "%-17s", "implementation");
 305
 306         for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++)
 307                 off += snprintf(buf + off, size - off, "%-16s",
 308                     raidz_gen_name[i]);
 309
 310         for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++)
 311                 off += snprintf(buf + off, size - off, "%-16s",
 312                     raidz_rec_name[i]);
 313
 314         (void) snprintf(buf + off, size - off, "\n");
 315
 316         return (0);
 317 }
 318
 319 static int
 320 raidz_math_kstat_data(char *buf, size_t size, void *data)
 321 {
 322         raidz_impl_kstat_t * fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
 323         raidz_impl_kstat_t * cstat = (raidz_impl_kstat_t *) data;
 324         ssize_t off = 0;
 325         int i;
 326
 327         ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN);
 328
 329         if (cstat == fstat) {
 330                 off += snprintf(buf + off, size - off, "%-17s", "fastest");
 331
 332                 for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) {
 333                         int id = fstat->gen[i];
 334                         off += snprintf(buf + off, size - off, "%-16s",
 335                             raidz_supp_impl[id]->name);
 336                 }
 337                 for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) {
 338                         int id = fstat->rec[i];
 339                         off += snprintf(buf + off, size - off, "%-16s",
 340                             raidz_supp_impl[id]->name);
 341                 }
 342         } else {
 343                 ptrdiff_t id = cstat - raidz_impl_kstats;
 344
 345                 off += snprintf(buf + off, size - off, "%-17s",
 346                     raidz_supp_impl[id]->name);
 347
 348                 for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++)
 349                         off += snprintf(buf + off, size - off, "%-16llu",
 350                             (u_longlong_t) cstat->gen[i]);
 351
 352                 for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++)
 353                         off += snprintf(buf + off, size - off, "%-16llu",
 354                             (u_longlong_t) cstat->rec[i]);
 355         }
 356
 357         (void) snprintf(buf + off, size - off, "\n");
 358
 359         return (0);
 360 }
 361
 362 static void *
 363 raidz_math_kstat_addr(kstat_t *ksp, loff_t n)
 364 {
 365         if (n <= raidz_supp_impl_cnt)
 366                 ksp->ks_private = (void *) (raidz_impl_kstats + n);
 367         else
 368                 ksp->ks_private = NULL;
 369
 370         return (ksp->ks_private);
 371 }
 372
 373 #define BENCH_D_COLS    (8ULL)
 374 #define BENCH_COLS      (BENCH_D_COLS + PARITY_PQR)
 375 #define BENCH_ZIO_SIZE  (1ULL << SPA_OLD_MAXBLOCKSHIFT) /* 128 kiB */
 376 #define BENCH_NS        MSEC2NSEC(25)                   /* 25ms */
 377
 378 typedef void (*benchmark_fn)(raidz_map_t *rm, const int fn);
 379
 380 static void
 381 benchmark_gen_impl(raidz_map_t *rm, const int fn)
 382 {
 383         (void) fn;
 384         vdev_raidz_generate_parity(rm);
 385 }
 386
 387 static void
 388 benchmark_rec_impl(raidz_map_t *rm, const int fn)
 389 {
 390         static const int rec_tgt[7][3] = {
 391                 {1, 2, 3},      /* rec_p:   bad QR & D[0]       */
 392                 {0, 2, 3},      /* rec_q:   bad PR & D[0]       */
 393                 {0, 1, 3},      /* rec_r:   bad PQ & D[0]       */
 394                 {2, 3, 4},      /* rec_pq:  bad R  & D[0][1]    */
 395                 {1, 3, 4},      /* rec_pr:  bad Q  & D[0][1]    */
 396                 {0, 3, 4},      /* rec_qr:  bad P  & D[0][1]    */
 397                 {3, 4, 5}       /* rec_pqr: bad    & D[0][1][2] */
 398         };
 399
 400         vdev_raidz_reconstruct(rm, rec_tgt[fn], 3);
 401 }
 402
 403 /*
 404  * Benchmarking of all supported implementations (raidz_supp_impl_cnt)
 405  * is performed by setting the rm_ops pointer and calling the top level
 406  * generate/reconstruct methods of bench_rm.
 407  */
 408 static void
 409 benchmark_raidz_impl(raidz_map_t *bench_rm, const int fn, benchmark_fn bench_fn)
 410 {
 411         uint64_t run_cnt, speed, best_speed = 0;
 412         hrtime_t t_start, t_diff;
 413         raidz_impl_ops_t *curr_impl;
 414         raidz_impl_kstat_t * fstat = &raidz_impl_kstats[raidz_supp_impl_cnt];
 415         int impl, i;
 416
 417         for (impl = 0; impl < raidz_supp_impl_cnt; impl++) {
 418                 /* set an implementation to benchmark */
 419                 curr_impl = raidz_supp_impl[impl];
 420                 bench_rm->rm_ops = curr_impl;
 421
 422                 run_cnt = 0;
 423                 t_start = gethrtime();
 424
 425                 do {
 426                         for (i = 0; i < 25; i++, run_cnt++)
 427                                 bench_fn(bench_rm, fn);
 428
 429                         t_diff = gethrtime() - t_start;
 430                 } while (t_diff < BENCH_NS);
 431
 432                 speed = run_cnt * BENCH_ZIO_SIZE * NANOSEC;
 433                 speed /= (t_diff * BENCH_COLS);
 434
 435                 if (bench_fn == benchmark_gen_impl)
 436                         raidz_impl_kstats[impl].gen[fn] = speed;
 437                 else
 438                         raidz_impl_kstats[impl].rec[fn] = speed;
 439
 440                 /* Update fastest implementation method */
 441                 if (speed > best_speed) {
 442                         best_speed = speed;
 443
 444                         if (bench_fn == benchmark_gen_impl) {
 445                                 fstat->gen[fn] = impl;
 446                                 vdev_raidz_fastest_impl.gen[fn] =
 447                                     curr_impl->gen[fn];
 448                         } else {
 449                                 fstat->rec[fn] = impl;
 450                                 vdev_raidz_fastest_impl.rec[fn] =
 451                                     curr_impl->rec[fn];
 452                         }
 453                 }
 454         }
 455 }
 456
 457 void
 458 vdev_raidz_math_init(void)
 459 {
 460         raidz_impl_ops_t *curr_impl;
 461         zio_t *bench_zio = NULL;
 462         raidz_map_t *bench_rm = NULL;
 463         uint64_t bench_parity;
 464         int i, c, fn;
 465
 466         /* move supported impl into raidz_supp_impl */
 467         for (i = 0, c = 0; i < ARRAY_SIZE(raidz_all_maths); i++) {
 468                 curr_impl = (raidz_impl_ops_t *) raidz_all_maths[i];
 469
 470                 /* initialize impl */
 471                 if (curr_impl->init)
 472                         curr_impl->init();
 473
 474                 if (curr_impl->is_supported())
 475                         raidz_supp_impl[c++] = (raidz_impl_ops_t *) curr_impl;
 476         }
 477         membar_producer();              /* complete raidz_supp_impl[] init */
 478         raidz_supp_impl_cnt = c;        /* number of supported impl */
 479
 480 #if !defined(_KERNEL)
 481         /* Skip benchmarking and use last implementation as fastest */
 482         memcpy(&vdev_raidz_fastest_impl, raidz_supp_impl[raidz_supp_impl_cnt-1],
 483             sizeof (vdev_raidz_fastest_impl));
 484         strcpy(vdev_raidz_fastest_impl.name, "fastest");
 485
 486         raidz_math_initialized = B_TRUE;
 487
 488         /* Use 'cycle' math selection method for userspace */
 489         VERIFY0(vdev_raidz_impl_set("cycle"));
 490         return;
 491 #endif
 492
 493         /* Fake an zio and run the benchmark on a warmed up buffer */
 494         bench_zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP);
 495         bench_zio->io_offset = 0;
 496         bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */
 497         bench_zio->io_abd = abd_alloc_linear(BENCH_ZIO_SIZE, B_TRUE);
 498         memset(abd_to_buf(bench_zio->io_abd), 0xAA, BENCH_ZIO_SIZE);
 499
 500         /* Benchmark parity generation methods */
 501         for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) {
 502                 bench_parity = fn + 1;
 503                 /* New raidz_map is needed for each generate_p/q/r */
 504                 bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT,
 505                     BENCH_D_COLS + bench_parity, bench_parity);
 506
 507                 benchmark_raidz_impl(bench_rm, fn, benchmark_gen_impl);
 508
 509                 vdev_raidz_map_free(bench_rm);
 510         }
 511
 512         /* Benchmark data reconstruction methods */
 513         bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT,
 514             BENCH_COLS, PARITY_PQR);
 515
 516         for (fn = 0; fn < RAIDZ_REC_NUM; fn++)
 517                 benchmark_raidz_impl(bench_rm, fn, benchmark_rec_impl);
 518
 519         vdev_raidz_map_free(bench_rm);
 520
 521         /* cleanup the bench zio */
 522         abd_free(bench_zio->io_abd);
 523         kmem_free(bench_zio, sizeof (zio_t));
 524
 525         /* install kstats for all impl */
 526         raidz_math_kstat = kstat_create("zfs", 0, "vdev_raidz_bench", "misc",
 527                 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
 528
 529         if (raidz_math_kstat != NULL) {
 530                 raidz_math_kstat->ks_data = NULL;
 531                 raidz_math_kstat->ks_ndata = UINT32_MAX;
 532                 kstat_set_raw_ops(raidz_math_kstat,
 533                     raidz_math_kstat_headers,
 534                     raidz_math_kstat_data,
 535                     raidz_math_kstat_addr);
 536                 kstat_install(raidz_math_kstat);
 537         }
 538
 539         /* Finish initialization */
 540         atomic_swap_32(&zfs_vdev_raidz_impl, user_sel_impl);
 541         raidz_math_initialized = B_TRUE;
 542 }
 543
 544 void
 545 vdev_raidz_math_fini(void)
 546 {
 547         raidz_impl_ops_t const *curr_impl;
 548         int i;
 549
 550         if (raidz_math_kstat != NULL) {
 551                 kstat_delete(raidz_math_kstat);
 552                 raidz_math_kstat = NULL;
 553         }
 554
 555         /* fini impl */
 556         for (i = 0; i < ARRAY_SIZE(raidz_all_maths); i++) {
 557                 curr_impl = raidz_all_maths[i];
 558                 if (curr_impl->fini)
 559                         curr_impl->fini();
 560         }
 561 }
 562
 563 static const struct {
 564         char    *name;
 565         uint32_t sel;
 566 } math_impl_opts[] = {
 567 #if !defined(_KERNEL)
 568                 { "cycle",      IMPL_CYCLE },
 569 #endif
 570                 { "fastest",    IMPL_FASTEST },
 571                 { "original",   IMPL_ORIGINAL },
 572                 { "scalar",     IMPL_SCALAR }
 573 };
 574
 575 /*
 576  * Function sets desired raidz implementation.
 577  *
 578  * If we are called before init(), user preference will be saved in
 579  * user_sel_impl, and applied in later init() call. This occurs when module
 580  * parameter is specified on module load. Otherwise, directly update
 581  * zfs_vdev_raidz_impl.
 582  *
 583  * @val         Name of raidz implementation to use
 584  * @param       Unused.
 585  */
 586 int
 587 vdev_raidz_impl_set(const char *val)
 588 {
 589         int err = -EINVAL;
 590         char req_name[RAIDZ_IMPL_NAME_MAX];
 591         uint32_t impl = RAIDZ_IMPL_READ(user_sel_impl);
 592         size_t i;
 593
 594         /* sanitize input */
 595         i = strnlen(val, RAIDZ_IMPL_NAME_MAX);
 596         if (i == 0 || i == RAIDZ_IMPL_NAME_MAX)
 597                 return (err);
 598
 599         strlcpy(req_name, val, RAIDZ_IMPL_NAME_MAX);
 600         while (i > 0 && !!isspace(req_name[i-1]))
 601                 i--;
 602         req_name[i] = '\0';
 603
 604         /* Check mandatory options */
 605         for (i = 0; i < ARRAY_SIZE(math_impl_opts); i++) {
 606                 if (strcmp(req_name, math_impl_opts[i].name) == 0) {
 607                         impl = math_impl_opts[i].sel;
 608                         err = 0;
 609                         break;
 610                 }
 611         }
 612
 613         /* check all supported impl if init() was already called */
 614         if (err != 0 && raidz_math_initialized) {
 615                 /* check all supported implementations */
 616                 for (i = 0; i < raidz_supp_impl_cnt; i++) {
 617                         if (strcmp(req_name, raidz_supp_impl[i]->name) == 0) {
 618                                 impl = i;
 619                                 err = 0;
 620                                 break;
 621                         }
 622                 }
 623         }
 624
 625         if (err == 0) {
 626                 if (raidz_math_initialized)
 627                         atomic_swap_32(&zfs_vdev_raidz_impl, impl);
 628                 else
 629                         atomic_swap_32(&user_sel_impl, impl);
 630         }
 631
 632         return (err);
 633 }
 634
 635 #if defined(_KERNEL) && defined(HAVE_SPL)
 636 #include <linux/mod_compat.h>
 637
 638 static int
 639 zfs_vdev_raidz_impl_set(const char *val, zfs_kernel_param_t *kp)
 640 {
 641         return (vdev_raidz_impl_set(val));
 642 }
 643
 644 static int
 645 zfs_vdev_raidz_impl_get(char *buffer, zfs_kernel_param_t *kp)
 646 {
 647         int i, cnt = 0;
 648         char *fmt;
 649         const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl);
 650
 651         ASSERT(raidz_math_initialized);
 652
 653         /* list mandatory options */
 654         for (i = 0; i < ARRAY_SIZE(math_impl_opts) - 2; i++) {
 655                 fmt = (impl == math_impl_opts[i].sel) ? "[%s] " : "%s ";
 656                 cnt += sprintf(buffer + cnt, fmt, math_impl_opts[i].name);
 657         }
 658
 659         /* list all supported implementations */
 660         for (i = 0; i < raidz_supp_impl_cnt; i++) {
 661                 fmt = (i == impl) ? "[%s] " : "%s ";
 662                 cnt += sprintf(buffer + cnt, fmt, raidz_supp_impl[i]->name);
 663         }
 664
 665         return (cnt);
 666 }
 667
 668 module_param_call(zfs_vdev_raidz_impl, zfs_vdev_raidz_impl_set,
 669         zfs_vdev_raidz_impl_get, NULL, 0644);
 670 MODULE_PARM_DESC(zfs_vdev_raidz_impl, "Select raidz implementation.");
 671 #endif