sys/cam/ctl/ctl_frontend_internal.c

   1 /*-
   2  * Copyright (c) 2004, 2005 Silicon Graphics International Corp.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice, this list of conditions, and the following disclaimer,
  10  *    without modification.
  11  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  12  *    substantially similar to the "NO WARRANTY" disclaimer below
  13  *    ("Disclaimer") and any redistribution must be conditioned upon
  14  *    including a substantially similar Disclaimer requirement for further
  15  *    binary redistribution.
  16  *
  17  * NO WARRANTY
  18  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  21  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  23  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  24  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  25  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  26  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  27  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  28  * POSSIBILITY OF SUCH DAMAGES.
  29  *
  30  * $Id: //depot/users/kenm/FreeBSD-test2/sys/cam/ctl/ctl_frontend_internal.c#5 $
  31  */
  32 /*
  33  * CTL kernel internal frontend target driver.  This allows kernel-level
  34  * clients to send commands into CTL.
  35  *
  36  * This has elements of a FETD (e.g. it has to set tag numbers, initiator,
  37  * port, target, and LUN) and elements of an initiator (LUN discovery and
  38  * probing, error recovery, command initiation).  Even though this has some
  39  * initiator type elements, this is not intended to be a full fledged
  40  * initiator layer.  It is only intended to send a limited number of
  41  * commands to a well known target layer.
  42  *
  43  * To be able to fulfill the role of a full initiator layer, it would need
  44  * a whole lot more functionality.
  45  *
  46  * Author: Ken Merry <ken@FreeBSD.org>
  47  *
  48  */
  49
  50 #include <sys/cdefs.h>
  51 __FBSDID("$FreeBSD$");
  52
  53 #include <sys/param.h>
  54 #include <sys/systm.h>
  55 #include <sys/kernel.h>
  56 #include <sys/types.h>
  57 #include <sys/malloc.h>
  58 #include <sys/module.h>
  59 #include <sys/lock.h>
  60 #include <sys/mutex.h>
  61 #include <sys/condvar.h>
  62 #include <sys/queue.h>
  63 #include <sys/sbuf.h>
  64 #include <sys/sysctl.h>
  65 #include <cam/scsi/scsi_all.h>
  66 #include <cam/scsi/scsi_da.h>
  67 #include <cam/ctl/ctl_io.h>
  68 #include <cam/ctl/ctl.h>
  69 #include <cam/ctl/ctl_frontend.h>
  70 #include <cam/ctl/ctl_frontend_internal.h>
  71 #include <cam/ctl/ctl_backend.h>
  72 #include <cam/ctl/ctl_ioctl.h>
  73 #include <cam/ctl/ctl_util.h>
  74 #include <cam/ctl/ctl_ha.h>
  75 #include <cam/ctl/ctl_private.h>
  76 #include <cam/ctl/ctl_mem_pool.h>
  77 #include <cam/ctl/ctl_debug.h>
  78 #include <cam/ctl/ctl_scsi_all.h>
  79 #include <cam/ctl/ctl_error.h>
  80
  81 /*
  82  * Task structure:
  83  *  - overall metatask, different potential metatask types (e.g. forced
  84  *    shutdown, gentle shutdown)
  85  *  - forced shutdown metatask:
  86  *     - states:  report luns, pending, done?
  87  *     - list of luns pending, with the relevant I/O for that lun attached.
  88  *       This would allow moving ahead on LUNs with no errors, and going
  89  *       into error recovery on LUNs with problems.  Per-LUN states might
  90  *       include inquiry, stop/offline, done.
  91  *
  92  * Use LUN enable for LUN list instead of getting it manually?  We'd still
  93  * need inquiry data for each LUN.
  94  *
  95  * How to handle processor LUN w.r.t. found/stopped counts?
  96  */
  97 #ifdef oldapi
  98 typedef enum {
  99         CFI_TASK_NONE,
 100         CFI_TASK_SHUTDOWN,
 101         CFI_TASK_STARTUP
 102 } cfi_tasktype;
 103
 104 struct cfi_task_startstop {
 105         int total_luns;
 106         int luns_complete;
 107         int luns_failed;
 108         cfi_cb_t callback;
 109         void *callback_arg;
 110         /* XXX KDM add more fields here */
 111 };
 112
 113 union cfi_taskinfo {
 114         struct cfi_task_startstop startstop;
 115 };
 116
 117 struct cfi_metatask {
 118         cfi_tasktype            tasktype;
 119         cfi_mt_status           status;
 120         union cfi_taskinfo      taskinfo;
 121         struct ctl_mem_element  *element;
 122         void                    *cfi_context;
 123         STAILQ_ENTRY(cfi_metatask) links;
 124 };
 125 #endif
 126
 127 typedef enum {
 128         CFI_ERR_RETRY           = 0x000,
 129         CFI_ERR_FAIL            = 0x001,
 130         CFI_ERR_LUN_RESET       = 0x002,
 131         CFI_ERR_MASK            = 0x0ff,
 132         CFI_ERR_NO_DECREMENT    = 0x100
 133 } cfi_error_action;
 134
 135 typedef enum {
 136         CFI_ERR_SOFT,
 137         CFI_ERR_HARD
 138 } cfi_error_policy;
 139
 140 typedef enum {
 141         CFI_LUN_INQUIRY,
 142         CFI_LUN_READCAPACITY,
 143         CFI_LUN_READCAPACITY_16,
 144         CFI_LUN_READY
 145 } cfi_lun_state;
 146
 147 struct cfi_lun {
 148         struct ctl_id target_id;
 149         int lun_id;
 150         struct scsi_inquiry_data inq_data;
 151         uint64_t num_blocks;
 152         uint32_t blocksize;
 153         int blocksize_powerof2;
 154         uint32_t cur_tag_num;
 155         cfi_lun_state state;
 156         struct ctl_mem_element *element;
 157         struct cfi_softc *softc;
 158         STAILQ_HEAD(, cfi_lun_io) io_list;
 159         STAILQ_ENTRY(cfi_lun) links;
 160 };
 161
 162 struct cfi_lun_io {
 163         struct cfi_lun *lun;
 164         struct cfi_metatask *metatask;
 165         cfi_error_policy policy;
 166         void (*done_function)(union ctl_io *io);
 167         union ctl_io *ctl_io;
 168         struct cfi_lun_io *orig_lun_io;
 169         STAILQ_ENTRY(cfi_lun_io) links;
 170 };
 171
 172 typedef enum {
 173         CFI_NONE        = 0x00,
 174         CFI_ONLINE      = 0x01,
 175 } cfi_flags;
 176
 177 struct cfi_softc {
 178         struct ctl_frontend fe;
 179         char fe_name[40];
 180         struct mtx lock;
 181         cfi_flags flags;
 182         STAILQ_HEAD(, cfi_lun) lun_list;
 183         STAILQ_HEAD(, cfi_metatask) metatask_list;
 184         struct ctl_mem_pool lun_pool;
 185         struct ctl_mem_pool metatask_pool;
 186 };
 187
 188 MALLOC_DEFINE(M_CTL_CFI, "ctlcfi", "CTL CFI");
 189
 190 static struct cfi_softc fetd_internal_softc;
 191
 192 int cfi_init(void);
 193 void cfi_shutdown(void) __unused;
 194 static void cfi_online(void *arg);
 195 static void cfi_offline(void *arg);
 196 static int cfi_targ_enable(void *arg, struct ctl_id targ_id);
 197 static int cfi_targ_disable(void *arg, struct ctl_id targ_id);
 198 static int cfi_lun_enable(void *arg, struct ctl_id target_id, int lun_id);
 199 static int cfi_lun_disable(void *arg, struct ctl_id target_id, int lun_id);
 200 static void cfi_datamove(union ctl_io *io);
 201 static cfi_error_action cfi_checkcond_parse(union ctl_io *io,
 202                                             struct cfi_lun_io *lun_io);
 203 static cfi_error_action cfi_error_parse(union ctl_io *io,
 204                                         struct cfi_lun_io *lun_io);
 205 static void cfi_init_io(union ctl_io *io, struct cfi_lun *lun,
 206                         struct cfi_metatask *metatask, cfi_error_policy policy,
 207                         int retries, struct cfi_lun_io *orig_lun_io,
 208                         void (*done_function)(union ctl_io *io));
 209 static void cfi_done(union ctl_io *io);
 210 static void cfi_lun_probe_done(union ctl_io *io);
 211 static void cfi_lun_probe(struct cfi_lun *lun, int have_lock);
 212 static void cfi_metatask_done(struct cfi_softc *softc,
 213                               struct cfi_metatask *metatask);
 214 static void cfi_metatask_bbr_errorparse(struct cfi_metatask *metatask,
 215                                         union ctl_io *io);
 216 static void cfi_metatask_io_done(union ctl_io *io);
 217 static void cfi_err_recovery_done(union ctl_io *io);
 218 static void cfi_lun_io_done(union ctl_io *io);
 219
 220 static int cfi_module_event_handler(module_t, int /*modeventtype_t*/, void *);
 221
 222 static moduledata_t cfi_moduledata = {
 223         "ctlcfi",
 224         cfi_module_event_handler,
 225         NULL
 226 };
 227
 228 DECLARE_MODULE(ctlcfi, cfi_moduledata, SI_SUB_CONFIGURE, SI_ORDER_FOURTH);
 229 MODULE_VERSION(ctlcfi, 1);
 230 MODULE_DEPEND(ctlcfi, ctl, 1, 1, 1);
 231
 232 int
 233 cfi_init(void)
 234 {
 235         struct cfi_softc *softc;
 236         struct ctl_frontend *fe;
 237         int retval;
 238
 239         softc = &fetd_internal_softc;
 240
 241         fe = &softc->fe;
 242
 243         retval = 0;
 244
 245         if (sizeof(struct cfi_lun_io) > CTL_PORT_PRIV_SIZE) {
 246                 printf("%s: size of struct cfi_lun_io %zd > "
 247                        "CTL_PORT_PRIV_SIZE %d\n", __func__,
 248                        sizeof(struct cfi_lun_io),
 249                        CTL_PORT_PRIV_SIZE);
 250         }
 251         memset(softc, 0, sizeof(*softc));
 252
 253         mtx_init(&softc->lock, "CTL frontend mutex", NULL, MTX_DEF);
 254         softc->flags |= CTL_FLAG_MASTER_SHELF;
 255
 256         STAILQ_INIT(&softc->lun_list);
 257         STAILQ_INIT(&softc->metatask_list);
 258         sprintf(softc->fe_name, "CTL internal");
 259         fe->port_type = CTL_PORT_INTERNAL;
 260         fe->num_requested_ctl_io = 100;
 261         fe->port_name = softc->fe_name;
 262         fe->port_online = cfi_online;
 263         fe->port_offline = cfi_offline;
 264         fe->onoff_arg = softc;
 265         fe->targ_enable = cfi_targ_enable;
 266         fe->targ_disable = cfi_targ_disable;
 267         fe->lun_enable = cfi_lun_enable;
 268         fe->lun_disable = cfi_lun_disable;
 269         fe->targ_lun_arg = softc;
 270         fe->fe_datamove = cfi_datamove;
 271         fe->fe_done = cfi_done;
 272         fe->max_targets = 15;
 273         fe->max_target_id = 15;
 274
 275         if (ctl_frontend_register(fe, (softc->flags & CTL_FLAG_MASTER_SHELF)) != 0)
 276         {
 277                 printf("%s: internal frontend registration failed\n", __func__);
 278                 retval = 1;
 279                 goto bailout;
 280         }
 281
 282         if (ctl_init_mem_pool(&softc->lun_pool,
 283                               sizeof(struct cfi_lun),
 284                               CTL_MEM_POOL_PERM_GROW, /*grow_inc*/ 3,
 285                               /* initial_pool_size */ CTL_MAX_LUNS) != 0) {
 286                 printf("%s: can't initialize LUN memory pool\n", __func__);
 287                 retval = 1;
 288                 goto bailout_error;
 289         }
 290
 291         if (ctl_init_mem_pool(&softc->metatask_pool,
 292                               sizeof(struct cfi_metatask),
 293                               CTL_MEM_POOL_PERM_GROW, /*grow_inc*/ 3,
 294                               /*initial_pool_size*/ 10) != 0) {
 295                 printf("%s: can't initialize metatask memory pool\n", __func__);
 296                 retval = 2;
 297                 goto bailout_error;
 298         }
 299 bailout:
 300
 301         return (0);
 302
 303 bailout_error:
 304
 305         switch (retval) {
 306         case 3:
 307                 ctl_shrink_mem_pool(&softc->metatask_pool);
 308                 /* FALLTHROUGH */
 309         case 2:
 310                 ctl_shrink_mem_pool(&softc->lun_pool);
 311                 /* FALLTHROUGH */
 312         case 1:
 313                 ctl_frontend_deregister(fe);
 314                 break;
 315         default:
 316                 break;
 317         }
 318
 319         return (ENOMEM);
 320 }
 321
 322 void
 323 cfi_shutdown(void)
 324 {
 325         struct cfi_softc *softc;
 326
 327         softc = &fetd_internal_softc;
 328
 329         /*
 330          * XXX KDM need to clear out any I/O pending on each LUN.
 331          */
 332         if (ctl_frontend_deregister(&softc->fe) != 0)
 333                 printf("%s: ctl_frontend_deregister() failed\n", __func__);
 334
 335         if (ctl_shrink_mem_pool(&softc->lun_pool) != 0)
 336                 printf("%s: error shrinking LUN pool\n", __func__);
 337
 338         if (ctl_shrink_mem_pool(&softc->metatask_pool) != 0)
 339                 printf("%s: error shrinking LUN pool\n", __func__);
 340 }
 341
 342 static int
 343 cfi_module_event_handler(module_t mod, int what, void *arg)
 344 {
 345
 346         switch (what) {
 347         case MOD_LOAD:
 348                 return (cfi_init());
 349         case MOD_UNLOAD:
 350                 return (EBUSY);
 351         default:
 352                 return (EOPNOTSUPP);
 353         }
 354 }
 355
 356 static void
 357 cfi_online(void *arg)
 358 {
 359         struct cfi_softc *softc;
 360         struct cfi_lun *lun;
 361
 362         softc = (struct cfi_softc *)arg;
 363
 364         softc->flags |= CFI_ONLINE;
 365
 366         /*
 367          * Go through and kick off the probe for each lun.  Should we check
 368          * the LUN flags here to determine whether or not to probe it?
 369          */
 370         mtx_lock(&softc->lock);
 371         STAILQ_FOREACH(lun, &softc->lun_list, links)
 372                 cfi_lun_probe(lun, /*have_lock*/ 1);
 373         mtx_unlock(&softc->lock);
 374 }
 375
 376 static void
 377 cfi_offline(void *arg)
 378 {
 379         struct cfi_softc *softc;
 380
 381         softc = (struct cfi_softc *)arg;
 382
 383         softc->flags &= ~CFI_ONLINE;
 384 }
 385
 386 static int
 387 cfi_targ_enable(void *arg, struct ctl_id targ_id)
 388 {
 389         return (0);
 390 }
 391
 392 static int
 393 cfi_targ_disable(void *arg, struct ctl_id targ_id)
 394 {
 395         return (0);
 396 }
 397
 398 static int
 399 cfi_lun_enable(void *arg, struct ctl_id target_id, int lun_id)
 400 {
 401         struct ctl_mem_element *element;
 402         struct cfi_softc *softc;
 403         struct cfi_lun *lun;
 404         int found;
 405
 406         softc = (struct cfi_softc *)arg;
 407
 408         found = 0;
 409         mtx_lock(&softc->lock);
 410         STAILQ_FOREACH(lun, &softc->lun_list, links) {
 411                 if ((lun->target_id.id == target_id.id)
 412                  && (lun->lun_id == lun_id)) {
 413                         found = 1;
 414                         break;
 415                 }
 416         }
 417         mtx_unlock(&softc->lock);
 418
 419         /*
 420          * If we already have this target/LUN, there is no reason to add
 421          * it to our lists again.
 422          */
 423         if (found != 0)
 424                 return (0);
 425
 426         element = ctl_alloc_mem_element(&softc->lun_pool, /*can_wait*/ 0);
 427
 428         if (element == NULL) {
 429                 printf("%s: unable to allocate LUN structure\n", __func__);
 430                 return (1);
 431         }
 432
 433         lun = (struct cfi_lun *)element->bytes;
 434
 435         lun->element = element;
 436         lun->target_id = target_id;
 437         lun->lun_id = lun_id;
 438         lun->cur_tag_num = 0;
 439         lun->state = CFI_LUN_INQUIRY;
 440         lun->softc = softc;
 441         STAILQ_INIT(&lun->io_list);
 442
 443         mtx_lock(&softc->lock);
 444         STAILQ_INSERT_TAIL(&softc->lun_list, lun, links);
 445         mtx_unlock(&softc->lock);
 446
 447         cfi_lun_probe(lun, /*have_lock*/ 0);
 448
 449         return (0);
 450 }
 451
 452 static int
 453 cfi_lun_disable(void *arg, struct ctl_id target_id, int lun_id)
 454 {
 455         struct cfi_softc *softc;
 456         struct cfi_lun *lun;
 457         int found;
 458
 459         softc = (struct cfi_softc *)arg;
 460
 461         found = 0;
 462
 463         /*
 464          * XXX KDM need to do an invalidate and then a free when any
 465          * pending I/O has completed.  Or do we?  CTL won't free a LUN
 466          * while any I/O is pending.  So we won't get this notification
 467          * unless any I/O we have pending on a LUN has completed.
 468          */
 469         mtx_lock(&softc->lock);
 470         STAILQ_FOREACH(lun, &softc->lun_list, links) {
 471                 if ((lun->target_id.id == target_id.id)
 472                  && (lun->lun_id == lun_id)) {
 473                         found = 1;
 474                         break;
 475                 }
 476         }
 477         if (found != 0)
 478                 STAILQ_REMOVE(&softc->lun_list, lun, cfi_lun, links);
 479
 480         mtx_unlock(&softc->lock);
 481
 482         if (found == 0) {
 483                 printf("%s: can't find target %ju lun %d\n", __func__,
 484                        (uintmax_t)target_id.id, lun_id);
 485                 return (1);
 486         }
 487
 488         ctl_free_mem_element(lun->element);
 489
 490         return (0);
 491 }
 492
 493 static void
 494 cfi_datamove(union ctl_io *io)
 495 {
 496         struct ctl_sg_entry *ext_sglist, *kern_sglist;
 497         struct ctl_sg_entry ext_entry, kern_entry;
 498         int ext_sglen, ext_sg_entries, kern_sg_entries;
 499         int ext_sg_start, ext_offset;
 500         int len_to_copy, len_copied;
 501         int kern_watermark, ext_watermark;
 502         int ext_sglist_malloced;
 503         struct ctl_scsiio *ctsio;
 504         int i, j;
 505
 506         ext_sglist_malloced = 0;
 507         ext_sg_start = 0;
 508         ext_offset = 0;
 509         ext_sglist = NULL;
 510
 511         CTL_DEBUG_PRINT(("%s\n", __func__));
 512
 513         ctsio = &io->scsiio;
 514
 515         /*
 516          * If this is the case, we're probably doing a BBR read and don't
 517          * actually need to transfer the data.  This will effectively
 518          * bit-bucket the data.
 519          */
 520         if (ctsio->ext_data_ptr == NULL)
 521                 goto bailout;
 522
 523         /*
 524          * To simplify things here, if we have a single buffer, stick it in
 525          * a S/G entry and just make it a single entry S/G list.
 526          */
 527         if (ctsio->io_hdr.flags & CTL_FLAG_EDPTR_SGLIST) {
 528                 int len_seen;
 529
 530                 ext_sglen = ctsio->ext_sg_entries * sizeof(*ext_sglist);
 531
 532                 ext_sglist = (struct ctl_sg_entry *)malloc(ext_sglen, M_CTL_CFI,
 533                                                            M_WAITOK);
 534                 ext_sglist_malloced = 1;
 535                 if (memcpy(ext_sglist, ctsio->ext_data_ptr, ext_sglen) != 0) {
 536                         ctl_set_internal_failure(ctsio,
 537                                                  /*sks_valid*/ 0,
 538                                                  /*retry_count*/ 0);
 539                         goto bailout;
 540                 }
 541                 ext_sg_entries = ctsio->ext_sg_entries;
 542                 len_seen = 0;
 543                 for (i = 0; i < ext_sg_entries; i++) {
 544                         if ((len_seen + ext_sglist[i].len) >=
 545                              ctsio->ext_data_filled) {
 546                                 ext_sg_start = i;
 547                                 ext_offset = ctsio->ext_data_filled - len_seen;
 548                                 break;
 549                         }
 550                         len_seen += ext_sglist[i].len;
 551                 }
 552         } else {
 553                 ext_sglist = &ext_entry;
 554                 ext_sglist->addr = ctsio->ext_data_ptr;
 555                 ext_sglist->len = ctsio->ext_data_len;
 556                 ext_sg_entries = 1;
 557                 ext_sg_start = 0;
 558                 ext_offset = ctsio->ext_data_filled;
 559         }
 560
 561         if (ctsio->kern_sg_entries > 0) {
 562                 kern_sglist = (struct ctl_sg_entry *)ctsio->kern_data_ptr;
 563                 kern_sg_entries = ctsio->kern_sg_entries;
 564         } else {
 565                 kern_sglist = &kern_entry;
 566                 kern_sglist->addr = ctsio->kern_data_ptr;
 567                 kern_sglist->len = ctsio->kern_data_len;
 568                 kern_sg_entries = 1;
 569         }
 570
 571
 572         kern_watermark = 0;
 573         ext_watermark = ext_offset;
 574         len_copied = 0;
 575         for (i = ext_sg_start, j = 0;
 576              i < ext_sg_entries && j < kern_sg_entries;) {
 577                 uint8_t *ext_ptr, *kern_ptr;
 578
 579                 len_to_copy = ctl_min(ext_sglist[i].len - ext_watermark,
 580                                       kern_sglist[j].len - kern_watermark);
 581
 582                 ext_ptr = (uint8_t *)ext_sglist[i].addr;
 583                 ext_ptr = ext_ptr + ext_watermark;
 584                 if (io->io_hdr.flags & CTL_FLAG_BUS_ADDR) {
 585                         /*
 586                          * XXX KDM fix this!
 587                          */
 588                         panic("need to implement bus address support");
 589 #if 0
 590                         kern_ptr = bus_to_virt(kern_sglist[j].addr);
 591 #endif
 592                 } else
 593                         kern_ptr = (uint8_t *)kern_sglist[j].addr;
 594                 kern_ptr = kern_ptr + kern_watermark;
 595
 596                 kern_watermark += len_to_copy;
 597                 ext_watermark += len_to_copy;
 598
 599                 if ((ctsio->io_hdr.flags & CTL_FLAG_DATA_MASK) ==
 600                      CTL_FLAG_DATA_IN) {
 601                         CTL_DEBUG_PRINT(("%s: copying %d bytes to user\n",
 602                                          __func__, len_to_copy));
 603                         CTL_DEBUG_PRINT(("%s: from %p to %p\n", __func__,
 604                                          kern_ptr, ext_ptr));
 605                         memcpy(ext_ptr, kern_ptr, len_to_copy);
 606                 } else {
 607                         CTL_DEBUG_PRINT(("%s: copying %d bytes from user\n",
 608                                          __func__, len_to_copy));
 609                         CTL_DEBUG_PRINT(("%s: from %p to %p\n", __func__,
 610                                          ext_ptr, kern_ptr));
 611                         memcpy(kern_ptr, ext_ptr, len_to_copy);
 612                 }
 613
 614                 len_copied += len_to_copy;
 615
 616                 if (ext_sglist[i].len == ext_watermark) {
 617                         i++;
 618                         ext_watermark = 0;
 619                 }
 620
 621                 if (kern_sglist[j].len == kern_watermark) {
 622                         j++;
 623                         kern_watermark = 0;
 624                 }
 625         }
 626
 627         ctsio->ext_data_filled += len_copied;
 628
 629         CTL_DEBUG_PRINT(("%s: ext_sg_entries: %d, kern_sg_entries: %d\n",
 630                          __func__, ext_sg_entries, kern_sg_entries));
 631         CTL_DEBUG_PRINT(("%s: ext_data_len = %d, kern_data_len = %d\n",
 632                          __func__, ctsio->ext_data_len, ctsio->kern_data_len));
 633
 634
 635         /* XXX KDM set residual?? */
 636 bailout:
 637
 638         if (ext_sglist_malloced != 0)
 639                 free(ext_sglist, M_CTL_CFI);
 640
 641         io->scsiio.be_move_done(io);
 642
 643         return;
 644 }
 645
 646 /*
 647  * For any sort of check condition, busy, etc., we just retry.  We do not
 648  * decrement the retry count for unit attention type errors.  These are
 649  * normal, and we want to save the retry count for "real" errors.  Otherwise,
 650  * we could end up with situations where a command will succeed in some
 651  * situations and fail in others, depending on whether a unit attention is
 652  * pending.  Also, some of our error recovery actions, most notably the
 653  * LUN reset action, will cause a unit attention.
 654  *
 655  * We can add more detail here later if necessary.
 656  */
 657 static cfi_error_action
 658 cfi_checkcond_parse(union ctl_io *io, struct cfi_lun_io *lun_io)
 659 {
 660         cfi_error_action error_action;
 661         int error_code, sense_key, asc, ascq;
 662
 663         /*
 664          * Default to retrying the command.
 665          */
 666         error_action = CFI_ERR_RETRY;
 667
 668         scsi_extract_sense_len(&io->scsiio.sense_data,
 669                                io->scsiio.sense_len,
 670                                &error_code,
 671                                &sense_key,
 672                                &asc,
 673                                &ascq,
 674                                /*show_errors*/ 1);
 675
 676         switch (error_code) {
 677         case SSD_DEFERRED_ERROR:
 678         case SSD_DESC_DEFERRED_ERROR:
 679                 error_action |= CFI_ERR_NO_DECREMENT;
 680                 break;
 681         case SSD_CURRENT_ERROR:
 682         case SSD_DESC_CURRENT_ERROR:
 683         default: {
 684                 switch (sense_key) {
 685                 case SSD_KEY_UNIT_ATTENTION:
 686                         error_action |= CFI_ERR_NO_DECREMENT;
 687                         break;
 688                 case SSD_KEY_HARDWARE_ERROR:
 689                         /*
 690                          * This is our generic "something bad happened"
 691                          * error code.  It often isn't recoverable.
 692                          */
 693                         if ((asc == 0x44) && (ascq == 0x00))
 694                                 error_action = CFI_ERR_FAIL;
 695                         break;
 696                 case SSD_KEY_NOT_READY:
 697                         /*
 698                          * If the LUN is powered down, there likely isn't
 699                          * much point in retrying right now.
 700                          */
 701                         if ((asc == 0x04) && (ascq == 0x02))
 702                                 error_action = CFI_ERR_FAIL;
 703                         /*
 704                          * If the LUN is offline, there probably isn't much
 705                          * point in retrying, either.
 706                          */
 707                         if ((asc == 0x04) && (ascq == 0x03))
 708                                 error_action = CFI_ERR_FAIL;
 709                         break;
 710                 }
 711         }
 712         }
 713
 714         return (error_action);
 715 }
 716
 717 static cfi_error_action
 718 cfi_error_parse(union ctl_io *io, struct cfi_lun_io *lun_io)
 719 {
 720         cfi_error_action error_action;
 721
 722         error_action = CFI_ERR_RETRY;
 723
 724         switch (io->io_hdr.io_type) {
 725         case CTL_IO_SCSI:
 726                 switch (io->io_hdr.status & CTL_STATUS_MASK) {
 727                 case CTL_SCSI_ERROR:
 728                         switch (io->scsiio.scsi_status) {
 729                         case SCSI_STATUS_RESERV_CONFLICT:
 730                                 /*
 731                                  * For a reservation conflict, we'll usually
 732                                  * want the hard error recovery policy, so
 733                                  * we'll reset the LUN.
 734                                  */
 735                                 if (lun_io->policy == CFI_ERR_HARD)
 736                                         error_action =
 737                                                 CFI_ERR_LUN_RESET;
 738                                 else
 739                                         error_action =
 740                                                 CFI_ERR_RETRY;
 741                                 break;
 742                         case SCSI_STATUS_CHECK_COND:
 743                         default:
 744                                 error_action = cfi_checkcond_parse(io, lun_io);
 745                                 break;
 746                         }
 747                         break;
 748                 default:
 749                         error_action = CFI_ERR_RETRY;
 750                         break;
 751                 }
 752                 break;
 753         case CTL_IO_TASK:
 754                 /*
 755                  * In theory task management commands shouldn't fail...
 756                  */
 757                 error_action = CFI_ERR_RETRY;
 758                 break;
 759         default:
 760                 printf("%s: invalid ctl_io type %d\n", __func__,
 761                        io->io_hdr.io_type);
 762                 panic("%s: invalid ctl_io type %d\n", __func__,
 763                       io->io_hdr.io_type);
 764                 break;
 765         }
 766
 767         return (error_action);
 768 }
 769
 770 static void
 771 cfi_init_io(union ctl_io *io, struct cfi_lun *lun,
 772             struct cfi_metatask *metatask, cfi_error_policy policy, int retries,
 773             struct cfi_lun_io *orig_lun_io,
 774             void (*done_function)(union ctl_io *io))
 775 {
 776         struct cfi_lun_io *lun_io;
 777
 778         io->io_hdr.nexus.initid.id = 7;
 779         io->io_hdr.nexus.targ_port = lun->softc->fe.targ_port;
 780         io->io_hdr.nexus.targ_target.id = lun->target_id.id;
 781         io->io_hdr.nexus.targ_lun = lun->lun_id;
 782         io->io_hdr.retries = retries;
 783         lun_io = (struct cfi_lun_io *)io->io_hdr.port_priv;
 784         io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = lun_io;
 785         lun_io->lun = lun;
 786         lun_io->metatask = metatask;
 787         lun_io->ctl_io = io;
 788         lun_io->policy = policy;
 789         lun_io->orig_lun_io = orig_lun_io;
 790         lun_io->done_function = done_function;
 791         /*
 792          * We only set the tag number for SCSI I/Os.  For task management
 793          * commands, the tag number is only really needed for aborts, so
 794          * the caller can set it if necessary.
 795          */
 796         switch (io->io_hdr.io_type) {
 797         case CTL_IO_SCSI:
 798                 io->scsiio.tag_num = lun->cur_tag_num++;
 799                 break;
 800         case CTL_IO_TASK:
 801         default:
 802                 break;
 803         }
 804 }
 805
 806 static void
 807 cfi_done(union ctl_io *io)
 808 {
 809         struct cfi_lun_io *lun_io;
 810         struct cfi_softc *softc;
 811         struct cfi_lun *lun;
 812
 813         lun_io = (struct cfi_lun_io *)
 814                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
 815
 816         lun = lun_io->lun;
 817         softc = lun->softc;
 818
 819         /*
 820          * Very minimal retry logic.  We basically retry if we got an error
 821          * back, and the retry count is greater than 0.  If we ever want
 822          * more sophisticated initiator type behavior, the CAM error
 823          * recovery code in ../common might be helpful.
 824          */
 825         if (((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS)
 826          && (io->io_hdr.retries > 0)) {
 827                 ctl_io_status old_status;
 828                 cfi_error_action error_action;
 829
 830                 error_action = cfi_error_parse(io, lun_io);
 831
 832                 switch (error_action & CFI_ERR_MASK) {
 833                 case CFI_ERR_FAIL:
 834                         goto done;
 835                         break; /* NOTREACHED */
 836                 case CFI_ERR_LUN_RESET: {
 837                         union ctl_io *new_io;
 838                         struct cfi_lun_io *new_lun_io;
 839
 840                         new_io = ctl_alloc_io(softc->fe.ctl_pool_ref);
 841                         if (new_io == NULL) {
 842                                 printf("%s: unable to allocate ctl_io for "
 843                                        "error recovery\n", __func__);
 844                                 goto done;
 845                         }
 846                         ctl_zero_io(new_io);
 847
 848                         new_io->io_hdr.io_type = CTL_IO_TASK;
 849                         new_io->taskio.task_action = CTL_TASK_LUN_RESET;
 850
 851                         cfi_init_io(new_io,
 852                                     /*lun*/ lun_io->lun,
 853                                     /*metatask*/ NULL,
 854                                     /*policy*/ CFI_ERR_SOFT,
 855                                     /*retries*/ 0,
 856                                     /*orig_lun_io*/lun_io,
 857                                     /*done_function*/ cfi_err_recovery_done);
 858
 859
 860                         new_lun_io = (struct cfi_lun_io *)
 861                                 new_io->io_hdr.port_priv;
 862
 863                         mtx_lock(&lun->softc->lock);
 864                         STAILQ_INSERT_TAIL(&lun->io_list, new_lun_io, links);
 865                         mtx_unlock(&lun->softc->lock);
 866
 867                         io = new_io;
 868                         break;
 869                 }
 870                 case CFI_ERR_RETRY:
 871                 default:
 872                         if ((error_action & CFI_ERR_NO_DECREMENT) == 0)
 873                                 io->io_hdr.retries--;
 874                         break;
 875                 }
 876
 877                 old_status = io->io_hdr.status;
 878                 io->io_hdr.status = CTL_STATUS_NONE;
 879 #if 0
 880                 io->io_hdr.flags &= ~CTL_FLAG_ALREADY_DONE;
 881 #endif
 882                 io->io_hdr.flags &= ~CTL_FLAG_ABORT;
 883                 io->io_hdr.flags &= ~CTL_FLAG_SENT_2OTHER_SC;
 884
 885                 if (ctl_queue(io) != CTL_RETVAL_COMPLETE) {
 886                         printf("%s: error returned from ctl_queue()!\n",
 887                                __func__);
 888                         io->io_hdr.status = old_status;
 889                 } else
 890                         return;
 891         }
 892 done:
 893         lun_io->done_function(io);
 894 }
 895
 896 static void
 897 cfi_lun_probe_done(union ctl_io *io)
 898 {
 899         struct cfi_lun *lun;
 900         struct cfi_lun_io *lun_io;
 901
 902         lun_io = (struct cfi_lun_io *)
 903                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
 904         lun = lun_io->lun;
 905
 906         switch (lun->state) {
 907         case CFI_LUN_INQUIRY: {
 908                 if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) {
 909                         /* print out something here?? */
 910                         printf("%s: LUN %d probe failed because inquiry "
 911                                "failed\n", __func__, lun->lun_id);
 912                         ctl_io_error_print(io, NULL);
 913                 } else {
 914
 915                         if (SID_TYPE(&lun->inq_data) != T_DIRECT) {
 916                                 char path_str[40];
 917
 918                                 lun->state = CFI_LUN_READY;
 919                                 ctl_scsi_path_string(io, path_str,
 920                                                      sizeof(path_str));
 921                                 printf("%s", path_str);
 922                                 scsi_print_inquiry(&lun->inq_data);
 923                         } else {
 924                                 lun->state = CFI_LUN_READCAPACITY;
 925                                 cfi_lun_probe(lun, /*have_lock*/ 0);
 926                         }
 927                 }
 928                 mtx_lock(&lun->softc->lock);
 929                 STAILQ_REMOVE(&lun->io_list, lun_io, cfi_lun_io, links);
 930                 mtx_unlock(&lun->softc->lock);
 931                 ctl_free_io(io);
 932                 break;
 933         }
 934         case CFI_LUN_READCAPACITY:
 935         case CFI_LUN_READCAPACITY_16: {
 936                 uint64_t maxlba;
 937                 uint32_t blocksize;
 938
 939                 maxlba = 0;
 940                 blocksize = 0;
 941
 942                 if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SUCCESS) {
 943                         printf("%s: LUN %d probe failed because READ CAPACITY "
 944                                "failed\n", __func__, lun->lun_id);
 945                         ctl_io_error_print(io, NULL);
 946                 } else {
 947
 948                         if (lun->state == CFI_LUN_READCAPACITY) {
 949                                 struct scsi_read_capacity_data *rdcap;
 950
 951                                 rdcap = (struct scsi_read_capacity_data *)
 952                                         io->scsiio.ext_data_ptr;
 953
 954                                 maxlba = scsi_4btoul(rdcap->addr);
 955                                 blocksize = scsi_4btoul(rdcap->length);
 956                                 if (blocksize == 0) {
 957                                         printf("%s: LUN %d has invalid "
 958                                                "blocksize 0, probe aborted\n",
 959                                                __func__, lun->lun_id);
 960                                 } else if (maxlba == 0xffffffff) {
 961                                         lun->state = CFI_LUN_READCAPACITY_16;
 962                                         cfi_lun_probe(lun, /*have_lock*/ 0);
 963                                 } else
 964                                         lun->state = CFI_LUN_READY;
 965                         } else {
 966                                 struct scsi_read_capacity_data_long *rdcap_long;
 967
 968                                 rdcap_long = (struct
 969                                         scsi_read_capacity_data_long *)
 970                                         io->scsiio.ext_data_ptr;
 971                                 maxlba = scsi_8btou64(rdcap_long->addr);
 972                                 blocksize = scsi_4btoul(rdcap_long->length);
 973
 974                                 if (blocksize == 0) {
 975                                         printf("%s: LUN %d has invalid "
 976                                                "blocksize 0, probe aborted\n",
 977                                                __func__, lun->lun_id);
 978                                 } else
 979                                         lun->state = CFI_LUN_READY;
 980                         }
 981                 }
 982
 983                 if (lun->state == CFI_LUN_READY) {
 984                         char path_str[40];
 985
 986                         lun->num_blocks = maxlba + 1;
 987                         lun->blocksize = blocksize;
 988
 989                         /*
 990                          * If this is true, the blocksize is a power of 2.
 991                          * We already checked for 0 above.
 992                          */
 993                         if (((blocksize - 1) & blocksize) == 0) {
 994                                 int i;
 995
 996                                 for (i = 0; i < 32; i++) {
 997                                         if ((blocksize & (1 << i)) != 0) {
 998                                                 lun->blocksize_powerof2 = i;
 999                                                 break;
1000                                         }
1001                                 }
1002                         }
1003                         ctl_scsi_path_string(io, path_str,sizeof(path_str));
1004                         printf("%s", path_str);
1005                         scsi_print_inquiry(&lun->inq_data);
1006                         printf("%s %ju blocks, blocksize %d\n", path_str,
1007                                (uintmax_t)maxlba + 1, blocksize);
1008                 }
1009                 mtx_lock(&lun->softc->lock);
1010                 STAILQ_REMOVE(&lun->io_list, lun_io, cfi_lun_io, links);
1011                 mtx_unlock(&lun->softc->lock);
1012                 free(io->scsiio.ext_data_ptr, M_CTL_CFI);
1013                 ctl_free_io(io);
1014                 break;
1015         }
1016         case CFI_LUN_READY:
1017         default:
1018                 mtx_lock(&lun->softc->lock);
1019                 /* How did we get here?? */
1020                 STAILQ_REMOVE(&lun->io_list, lun_io, cfi_lun_io, links);
1021                 mtx_unlock(&lun->softc->lock);
1022                 ctl_free_io(io);
1023                 break;
1024         }
1025 }
1026
1027 static void
1028 cfi_lun_probe(struct cfi_lun *lun, int have_lock)
1029 {
1030
1031         if (have_lock == 0)
1032                 mtx_lock(&lun->softc->lock);
1033         if ((lun->softc->flags & CFI_ONLINE) == 0) {
1034                 if (have_lock == 0)
1035                         mtx_unlock(&lun->softc->lock);
1036                 return;
1037         }
1038         if (have_lock == 0)
1039                 mtx_unlock(&lun->softc->lock);
1040
1041         switch (lun->state) {
1042         case CFI_LUN_INQUIRY: {
1043                 struct cfi_lun_io *lun_io;
1044                 union ctl_io *io;
1045
1046                 io = ctl_alloc_io(lun->softc->fe.ctl_pool_ref);
1047                 if (io == NULL) {
1048                         printf("%s: unable to alloc ctl_io for target %ju "
1049                                "lun %d probe\n", __func__,
1050                                (uintmax_t)lun->target_id.id, lun->lun_id);
1051                         return;
1052                 }
1053                 ctl_scsi_inquiry(io,
1054                                  /*data_ptr*/(uint8_t *)&lun->inq_data,
1055                                  /*data_len*/ sizeof(lun->inq_data),
1056                                  /*byte2*/ 0,
1057                                  /*page_code*/ 0,
1058                                  /*tag_type*/ CTL_TAG_SIMPLE,
1059                                  /*control*/ 0);
1060
1061                 cfi_init_io(io,
1062                             /*lun*/ lun,
1063                             /*metatask*/ NULL,
1064                             /*policy*/ CFI_ERR_SOFT,
1065                             /*retries*/ 5,
1066                             /*orig_lun_io*/ NULL,
1067                             /*done_function*/
1068                             cfi_lun_probe_done);
1069
1070                 lun_io = (struct cfi_lun_io *)io->io_hdr.port_priv;
1071
1072                 if (have_lock == 0)
1073                         mtx_lock(&lun->softc->lock);
1074                 STAILQ_INSERT_TAIL(&lun->io_list, lun_io, links);
1075                 if (have_lock == 0)
1076                         mtx_unlock(&lun->softc->lock);
1077
1078                 if (ctl_queue(io) != CTL_RETVAL_COMPLETE) {
1079                         printf("%s: error returned from ctl_queue()!\n",
1080                                __func__);
1081                         STAILQ_REMOVE(&lun->io_list, lun_io,
1082                                       cfi_lun_io, links);
1083                         ctl_free_io(io);
1084                 }
1085                 break;
1086         }
1087         case CFI_LUN_READCAPACITY:
1088         case CFI_LUN_READCAPACITY_16: {
1089                 struct cfi_lun_io *lun_io;
1090                 uint8_t *dataptr;
1091                 union ctl_io *io;
1092
1093                 io = ctl_alloc_io(lun->softc->fe.ctl_pool_ref);
1094                 if (io == NULL) {
1095                         printf("%s: unable to alloc ctl_io for target %ju "
1096                                "lun %d probe\n", __func__,
1097                                (uintmax_t)lun->target_id.id, lun->lun_id);
1098                         return;
1099                 }
1100
1101                 dataptr = malloc(sizeof(struct scsi_read_capacity_data_long),
1102                                  M_CTL_CFI, M_NOWAIT);
1103                 if (dataptr == NULL) {
1104                         printf("%s: unable to allocate SCSI read capacity "
1105                                "buffer for target %ju lun %d\n", __func__,
1106                                (uintmax_t)lun->target_id.id, lun->lun_id);
1107                         return;
1108                 }
1109                 if (lun->state == CFI_LUN_READCAPACITY) {
1110                         ctl_scsi_read_capacity(io,
1111                                 /*data_ptr*/ dataptr,
1112                                 /*data_len*/
1113                                 sizeof(struct scsi_read_capacity_data_long),
1114                                 /*addr*/ 0,
1115                                 /*reladr*/ 0,
1116                                 /*pmi*/ 0,
1117                                 /*tag_type*/ CTL_TAG_SIMPLE,
1118                                 /*control*/ 0);
1119                 } else {
1120                         ctl_scsi_read_capacity_16(io,
1121                                 /*data_ptr*/ dataptr,
1122                                 /*data_len*/
1123                                 sizeof(struct scsi_read_capacity_data_long),
1124                                 /*addr*/ 0,
1125                                 /*reladr*/ 0,
1126                                 /*pmi*/ 0,
1127                                 /*tag_type*/ CTL_TAG_SIMPLE,
1128                                 /*control*/ 0);
1129                 }
1130                 cfi_init_io(io,
1131                             /*lun*/ lun,
1132                             /*metatask*/ NULL,
1133                             /*policy*/ CFI_ERR_SOFT,
1134                             /*retries*/ 7,
1135                             /*orig_lun_io*/ NULL,
1136                             /*done_function*/ cfi_lun_probe_done);
1137
1138                 lun_io = (struct cfi_lun_io *)io->io_hdr.port_priv;
1139
1140                 if (have_lock == 0)
1141                         mtx_lock(&lun->softc->lock);
1142                 STAILQ_INSERT_TAIL(&lun->io_list, lun_io, links);
1143                 if (have_lock == 0)
1144                         mtx_unlock(&lun->softc->lock);
1145
1146                 if (ctl_queue(io) != CTL_RETVAL_COMPLETE) {
1147                         printf("%s: error returned from ctl_queue()!\n",
1148                                __func__);
1149                         STAILQ_REMOVE(&lun->io_list, lun_io,
1150                                       cfi_lun_io, links);
1151                         free(dataptr, M_CTL_CFI);
1152                         ctl_free_io(io);
1153                 }
1154                 break;
1155         }
1156         case CFI_LUN_READY:
1157         default:
1158                 /* Why were we called? */
1159                 break;
1160         }
1161 }
1162
1163 static void
1164 cfi_metatask_done(struct cfi_softc *softc, struct cfi_metatask *metatask)
1165 {
1166         mtx_lock(&softc->lock);
1167         STAILQ_REMOVE(&softc->metatask_list, metatask, cfi_metatask, links);
1168         mtx_unlock(&softc->lock);
1169
1170         /*
1171          * Return status to the caller.  Caller allocated storage, and is
1172          * responsible for calling cfi_free_metatask to release it once
1173          * they've seen the status.
1174          */
1175         metatask->callback(metatask->callback_arg, metatask);
1176 }
1177
1178 static void
1179 cfi_metatask_bbr_errorparse(struct cfi_metatask *metatask, union ctl_io *io)
1180 {
1181         int error_code, sense_key, asc, ascq;
1182
1183         if (metatask->tasktype != CFI_TASK_BBRREAD)
1184                 return;
1185
1186         if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS) {
1187                 metatask->status = CFI_MT_SUCCESS;
1188                 metatask->taskinfo.bbrread.status = CFI_BBR_SUCCESS;
1189                 return;
1190         }
1191
1192         if ((io->io_hdr.status & CTL_STATUS_MASK) != CTL_SCSI_ERROR) {
1193                 metatask->status = CFI_MT_ERROR;
1194                 metatask->taskinfo.bbrread.status = CFI_BBR_ERROR;
1195                 return;
1196         }
1197
1198         metatask->taskinfo.bbrread.scsi_status = io->scsiio.scsi_status;
1199         memcpy(&metatask->taskinfo.bbrread.sense_data, &io->scsiio.sense_data,
1200                ctl_min(sizeof(metatask->taskinfo.bbrread.sense_data),
1201                        sizeof(io->scsiio.sense_data)));
1202
1203         if (io->scsiio.scsi_status == SCSI_STATUS_RESERV_CONFLICT) {
1204                 metatask->status = CFI_MT_ERROR;
1205                 metatask->taskinfo.bbrread.status = CFI_BBR_RESERV_CONFLICT;
1206                 return;
1207         }
1208
1209         if (io->scsiio.scsi_status != SCSI_STATUS_CHECK_COND) {
1210                 metatask->status = CFI_MT_ERROR;
1211                 metatask->taskinfo.bbrread.status = CFI_BBR_SCSI_ERROR;
1212                 return;
1213         }
1214
1215         scsi_extract_sense_len(&io->scsiio.sense_data,
1216                                io->scsiio.sense_len,
1217                                &error_code,
1218                                &sense_key,
1219                                &asc,
1220                                &ascq,
1221                                /*show_errors*/ 1);
1222
1223         switch (error_code) {
1224         case SSD_DEFERRED_ERROR:
1225         case SSD_DESC_DEFERRED_ERROR:
1226                 metatask->status = CFI_MT_ERROR;
1227                 metatask->taskinfo.bbrread.status = CFI_BBR_SCSI_ERROR;
1228                 break;
1229         case SSD_CURRENT_ERROR:
1230         case SSD_DESC_CURRENT_ERROR:
1231         default: {
1232                 struct scsi_sense_data *sense;
1233
1234                 sense = &io->scsiio.sense_data;
1235
1236                 if ((asc == 0x04) && (ascq == 0x02)) {
1237                         metatask->status = CFI_MT_ERROR;
1238                         metatask->taskinfo.bbrread.status = CFI_BBR_LUN_STOPPED;
1239                 } else if ((asc == 0x04) && (ascq == 0x03)) {
1240                         metatask->status = CFI_MT_ERROR;
1241                         metatask->taskinfo.bbrread.status =
1242                                 CFI_BBR_LUN_OFFLINE_CTL;
1243                 } else if ((asc == 0x44) && (ascq == 0x00)) {
1244 #ifdef NEEDTOPORT
1245                         if (sense->sense_key_spec[0] & SSD_SCS_VALID) {
1246                                 uint16_t retry_count;
1247
1248                                 retry_count = sense->sense_key_spec[1] << 8 |
1249                                               sense->sense_key_spec[2];
1250                                 if (((retry_count & 0xf000) == CSC_RAIDCORE)
1251                                  && ((retry_count & 0x0f00) == CSC_SHELF_SW)
1252                                  && ((retry_count & 0xff) ==
1253                                       RC_STS_DEVICE_OFFLINE)) {
1254                                         metatask->status = CFI_MT_ERROR;
1255                                         metatask->taskinfo.bbrread.status =
1256                                                 CFI_BBR_LUN_OFFLINE_RC;
1257                                 } else {
1258                                         metatask->status = CFI_MT_ERROR;
1259                                         metatask->taskinfo.bbrread.status =
1260                                                 CFI_BBR_SCSI_ERROR;
1261                                 }
1262                         } else {
1263 #endif /* NEEDTOPORT */
1264                                 metatask->status = CFI_MT_ERROR;
1265                                 metatask->taskinfo.bbrread.status =
1266                                         CFI_BBR_SCSI_ERROR;
1267 #ifdef NEEDTOPORT
1268                         }
1269 #endif
1270                 } else {
1271                         metatask->status = CFI_MT_ERROR;
1272                         metatask->taskinfo.bbrread.status = CFI_BBR_SCSI_ERROR;
1273                 }
1274                 break;
1275         }
1276         }
1277 }
1278
1279 static void
1280 cfi_metatask_io_done(union ctl_io *io)
1281 {
1282         struct cfi_lun_io *lun_io;
1283         struct cfi_metatask *metatask;
1284         struct cfi_softc *softc;
1285         struct cfi_lun *lun;
1286
1287         lun_io = (struct cfi_lun_io *)
1288                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1289
1290         lun = lun_io->lun;
1291         softc = lun->softc;
1292
1293         metatask = lun_io->metatask;
1294
1295         switch (metatask->tasktype) {
1296         case CFI_TASK_STARTUP:
1297         case CFI_TASK_SHUTDOWN: {
1298                 int failed, done, is_start;
1299
1300                 failed = 0;
1301                 done = 0;
1302                 if (metatask->tasktype == CFI_TASK_STARTUP)
1303                         is_start = 1;
1304                 else
1305                         is_start = 0;
1306
1307                 mtx_lock(&softc->lock);
1308                 if ((io->io_hdr.status & CTL_STATUS_MASK) == CTL_SUCCESS)
1309                         metatask->taskinfo.startstop.luns_complete++;
1310                 else {
1311                         metatask->taskinfo.startstop.luns_failed++;
1312                         failed = 1;
1313                 }
1314                 if ((metatask->taskinfo.startstop.luns_complete +
1315                      metatask->taskinfo.startstop.luns_failed) >=
1316                      metatask->taskinfo.startstop.total_luns)
1317                         done = 1;
1318
1319                 mtx_unlock(&softc->lock);
1320
1321                 if (failed != 0) {
1322                         printf("%s: LUN %d %s request failed\n", __func__,
1323                                lun_io->lun->lun_id, (is_start == 1) ? "start" :
1324                                "stop");
1325                         ctl_io_error_print(io, &lun_io->lun->inq_data);
1326                 }
1327                 if (done != 0) {
1328                         if (metatask->taskinfo.startstop.luns_failed > 0)
1329                                 metatask->status = CFI_MT_ERROR;
1330                         else
1331                                 metatask->status = CFI_MT_SUCCESS;
1332                         cfi_metatask_done(softc, metatask);
1333                 }
1334                 mtx_lock(&softc->lock);
1335                 STAILQ_REMOVE(&lun->io_list, lun_io, cfi_lun_io, links);
1336                 mtx_unlock(&softc->lock);
1337
1338                 ctl_free_io(io);
1339                 break;
1340         }
1341         case CFI_TASK_BBRREAD: {
1342                 /*
1343                  * Translate the SCSI error into an enumeration.
1344                  */
1345                 cfi_metatask_bbr_errorparse(metatask, io);
1346
1347                 mtx_lock(&softc->lock);
1348                 STAILQ_REMOVE(&lun->io_list, lun_io, cfi_lun_io, links);
1349                 mtx_unlock(&softc->lock);
1350
1351                 ctl_free_io(io);
1352
1353                 cfi_metatask_done(softc, metatask);
1354                 break;
1355         }
1356         default:
1357                 /*
1358                  * This shouldn't happen.
1359                  */
1360                 mtx_lock(&softc->lock);
1361                 STAILQ_REMOVE(&lun->io_list, lun_io, cfi_lun_io, links);
1362                 mtx_unlock(&softc->lock);
1363
1364                 ctl_free_io(io);
1365                 break;
1366         }
1367 }
1368
1369 static void
1370 cfi_err_recovery_done(union ctl_io *io)
1371 {
1372         struct cfi_lun_io *lun_io, *orig_lun_io;
1373         struct cfi_lun *lun;
1374         union ctl_io *orig_io;
1375
1376         lun_io = (struct cfi_lun_io *)io->io_hdr.port_priv;
1377         orig_lun_io = lun_io->orig_lun_io;
1378         orig_io = orig_lun_io->ctl_io;
1379         lun = lun_io->lun;
1380
1381         if (io->io_hdr.status != CTL_SUCCESS) {
1382                 printf("%s: error recovery action failed.  Original "
1383                        "error:\n", __func__);
1384
1385                 ctl_io_error_print(orig_lun_io->ctl_io, &lun->inq_data);
1386
1387                 printf("%s: error from error recovery action:\n", __func__);
1388
1389                 ctl_io_error_print(io, &lun->inq_data);
1390
1391                 printf("%s: trying original command again...\n", __func__);
1392         }
1393
1394         mtx_lock(&lun->softc->lock);
1395         STAILQ_REMOVE(&lun->io_list, lun_io, cfi_lun_io, links);
1396         mtx_unlock(&lun->softc->lock);
1397         ctl_free_io(io);
1398
1399         orig_io->io_hdr.retries--;
1400         orig_io->io_hdr.status = CTL_STATUS_NONE;
1401
1402         if (ctl_queue(orig_io) != CTL_RETVAL_COMPLETE) {
1403                 printf("%s: error returned from ctl_queue()!\n", __func__);
1404                 STAILQ_REMOVE(&lun->io_list, orig_lun_io,
1405                               cfi_lun_io, links);
1406                 ctl_free_io(orig_io);
1407         }
1408 }
1409
1410 static void
1411 cfi_lun_io_done(union ctl_io *io)
1412 {
1413         struct cfi_lun *lun;
1414         struct cfi_lun_io *lun_io;
1415
1416         lun_io = (struct cfi_lun_io *)
1417                 io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr;
1418         lun = lun_io->lun;
1419
1420         if (lun_io->metatask == NULL) {
1421                 printf("%s: I/O has no metatask pointer, discarding\n",
1422                        __func__);
1423                 STAILQ_REMOVE(&lun->io_list, lun_io, cfi_lun_io, links);
1424                 ctl_free_io(io);
1425                 return;
1426         }
1427         cfi_metatask_io_done(io);
1428 }
1429
1430 void
1431 cfi_action(struct cfi_metatask *metatask)
1432 {
1433         struct cfi_softc *softc;
1434
1435         softc = &fetd_internal_softc;
1436
1437         mtx_lock(&softc->lock);
1438
1439         STAILQ_INSERT_TAIL(&softc->metatask_list, metatask, links);
1440
1441         if ((softc->flags & CFI_ONLINE) == 0) {
1442                 mtx_unlock(&softc->lock);
1443                 metatask->status = CFI_MT_PORT_OFFLINE;
1444                 cfi_metatask_done(softc, metatask);
1445                 return;
1446         } else
1447                 mtx_unlock(&softc->lock);
1448
1449         switch (metatask->tasktype) {
1450         case CFI_TASK_STARTUP:
1451         case CFI_TASK_SHUTDOWN: {
1452                 union ctl_io *io;
1453                 int da_luns, ios_allocated, do_start;
1454                 struct cfi_lun *lun;
1455                 STAILQ_HEAD(, ctl_io_hdr) tmp_io_list;
1456
1457                 da_luns = 0;
1458                 ios_allocated = 0;
1459                 STAILQ_INIT(&tmp_io_list);
1460
1461                 if (metatask->tasktype == CFI_TASK_STARTUP)
1462                         do_start = 1;
1463                 else
1464                         do_start = 0;
1465
1466                 mtx_lock(&softc->lock);
1467                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
1468                         if (lun->state != CFI_LUN_READY)
1469                                 continue;
1470
1471                         if (SID_TYPE(&lun->inq_data) != T_DIRECT)
1472                                 continue;
1473                         da_luns++;
1474                         io = ctl_alloc_io(softc->fe.ctl_pool_ref);
1475                         if (io != NULL) {
1476                                 ios_allocated++;
1477                                 STAILQ_INSERT_TAIL(&tmp_io_list, &io->io_hdr,
1478                                                    links);
1479                         }
1480                 }
1481
1482                 if (ios_allocated < da_luns) {
1483                         printf("%s: error allocating ctl_io for %s\n",
1484                                __func__, (do_start == 1) ? "startup" :
1485                                "shutdown");
1486                         da_luns = ios_allocated;
1487                 }
1488
1489                 metatask->taskinfo.startstop.total_luns = da_luns;
1490
1491                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
1492                         struct cfi_lun_io *lun_io;
1493
1494                         if (lun->state != CFI_LUN_READY)
1495                                 continue;
1496
1497                         if (SID_TYPE(&lun->inq_data) != T_DIRECT)
1498                                 continue;
1499
1500                         io = (union ctl_io *)STAILQ_FIRST(&tmp_io_list);
1501                         if (io == NULL)
1502                                 break;
1503
1504                         STAILQ_REMOVE(&tmp_io_list, &io->io_hdr, ctl_io_hdr,
1505                                       links);
1506
1507                         ctl_scsi_start_stop(io,
1508                                             /*start*/ do_start,
1509                                             /*load_eject*/ 0,
1510                                             /*immediate*/ 0,
1511                                             /*power_conditions*/
1512                                             SSS_PC_START_VALID,
1513                                             /*onoffline*/ 1,
1514                                             /*ctl_tag_type*/ CTL_TAG_ORDERED,
1515                                             /*control*/ 0);
1516
1517                         cfi_init_io(io,
1518                                     /*lun*/ lun,
1519                                     /*metatask*/ metatask,
1520                                     /*policy*/ CFI_ERR_HARD,
1521                                     /*retries*/ 3,
1522                                     /*orig_lun_io*/ NULL,
1523                                     /*done_function*/ cfi_lun_io_done);
1524
1525                         lun_io = (struct cfi_lun_io *) io->io_hdr.port_priv;
1526
1527                         STAILQ_INSERT_TAIL(&lun->io_list, lun_io, links);
1528
1529                         if (ctl_queue(io) != CTL_RETVAL_COMPLETE) {
1530                                 printf("%s: error returned from ctl_queue()!\n",
1531                                        __func__);
1532                                 STAILQ_REMOVE(&lun->io_list, lun_io,
1533                                               cfi_lun_io, links);
1534                                 ctl_free_io(io);
1535                                 metatask->taskinfo.startstop.total_luns--;
1536                         }
1537                 }
1538
1539                 if (STAILQ_FIRST(&tmp_io_list) != NULL) {
1540                         printf("%s: error: tmp_io_list != NULL\n", __func__);
1541                         for (io = (union ctl_io *)STAILQ_FIRST(&tmp_io_list);
1542                              io != NULL;
1543                              io = (union ctl_io *)STAILQ_FIRST(&tmp_io_list)) {
1544                                 STAILQ_REMOVE(&tmp_io_list, &io->io_hdr,
1545                                               ctl_io_hdr, links);
1546                                 ctl_free_io(io);
1547                         }
1548                 }
1549                 mtx_unlock(&softc->lock);
1550
1551                 break;
1552         }
1553         case CFI_TASK_BBRREAD: {
1554                 union ctl_io *io;
1555                 struct cfi_lun *lun;
1556                 struct cfi_lun_io *lun_io;
1557                 cfi_bbrread_status status;
1558                 int req_lun_num;
1559                 uint32_t num_blocks;
1560
1561                 status = CFI_BBR_SUCCESS;
1562
1563                 req_lun_num = metatask->taskinfo.bbrread.lun_num;
1564
1565                 mtx_lock(&softc->lock);
1566                 STAILQ_FOREACH(lun, &softc->lun_list, links) {
1567                         if (lun->lun_id != req_lun_num)
1568                                 continue;
1569                         if (lun->state != CFI_LUN_READY) {
1570                                 status = CFI_BBR_LUN_UNCONFIG;
1571                                 break;
1572                         } else
1573                                 break;
1574                 }
1575
1576                 if (lun == NULL)
1577                         status = CFI_BBR_NO_LUN;
1578
1579                 if (status != CFI_BBR_SUCCESS) {
1580                         metatask->status = CFI_MT_ERROR;
1581                         metatask->taskinfo.bbrread.status = status;
1582                         mtx_unlock(&softc->lock);
1583                         cfi_metatask_done(softc, metatask);
1584                         break;
1585                 }
1586
1587                 /*
1588                  * Convert the number of bytes given into blocks and check
1589                  * that the number of bytes is a multiple of the blocksize.
1590                  * CTL will verify that the LBA is okay.
1591                  */
1592                 if (lun->blocksize_powerof2 != 0) {
1593                         if ((metatask->taskinfo.bbrread.len &
1594                             (lun->blocksize - 1)) != 0) {
1595                                 metatask->status = CFI_MT_ERROR;
1596                                 metatask->taskinfo.bbrread.status =
1597                                         CFI_BBR_BAD_LEN;
1598                                 cfi_metatask_done(softc, metatask);
1599                                 break;
1600                         }
1601
1602                         num_blocks = metatask->taskinfo.bbrread.len >>
1603                                 lun->blocksize_powerof2;
1604                 } else {
1605                         /*
1606                          * XXX KDM this could result in floating point
1607                          * division, which isn't supported in the kernel on
1608                          * x86 at least.
1609                          */
1610                         if ((metatask->taskinfo.bbrread.len %
1611                              lun->blocksize) != 0) {
1612                                 metatask->status = CFI_MT_ERROR;
1613                                 metatask->taskinfo.bbrread.status =
1614                                         CFI_BBR_BAD_LEN;
1615                                 cfi_metatask_done(softc, metatask);
1616                                 break;
1617                         }
1618
1619                         /*
1620                          * XXX KDM this could result in floating point
1621                          * division in some cases.
1622                          */
1623                         num_blocks = metatask->taskinfo.bbrread.len /
1624                                 lun->blocksize;
1625
1626                 }
1627
1628                 io = ctl_alloc_io(softc->fe.ctl_pool_ref);
1629                 if (io == NULL) {
1630                         metatask->status = CFI_MT_ERROR;
1631                         metatask->taskinfo.bbrread.status = CFI_BBR_NO_MEM;
1632                         mtx_unlock(&softc->lock);
1633                         cfi_metatask_done(softc, metatask);
1634                         break;
1635                 }
1636
1637                 /*
1638                  * XXX KDM need to do a read capacity to get the blocksize
1639                  * for this device.
1640                  */
1641                 ctl_scsi_read_write(io,
1642                                     /*data_ptr*/ NULL,
1643                                     /*data_len*/ metatask->taskinfo.bbrread.len,
1644                                     /*read_op*/ 1,
1645                                     /*byte2*/ 0,
1646                                     /*minimum_cdb_size*/ 0,
1647                                     /*lba*/ metatask->taskinfo.bbrread.lba,
1648                                     /*num_blocks*/ num_blocks,
1649                                     /*tag_type*/ CTL_TAG_SIMPLE,
1650                                     /*control*/ 0);
1651
1652                 cfi_init_io(io,
1653                             /*lun*/ lun,
1654                             /*metatask*/ metatask,
1655                             /*policy*/ CFI_ERR_SOFT,
1656                             /*retries*/ 3,
1657                             /*orig_lun_io*/ NULL,
1658                             /*done_function*/ cfi_lun_io_done);
1659
1660                 lun_io = (struct cfi_lun_io *)io->io_hdr.port_priv;
1661
1662                 STAILQ_INSERT_TAIL(&lun->io_list, lun_io, links);
1663
1664                 if (ctl_queue(io) != CTL_RETVAL_COMPLETE) {
1665                         printf("%s: error returned from ctl_queue()!\n",
1666                                __func__);
1667                         STAILQ_REMOVE(&lun->io_list, lun_io, cfi_lun_io, links);
1668                         ctl_free_io(io);
1669                         metatask->status = CFI_MT_ERROR;
1670                         metatask->taskinfo.bbrread.status = CFI_BBR_ERROR;
1671                         mtx_unlock(&softc->lock);
1672                         cfi_metatask_done(softc, metatask);
1673                         break;
1674                 }
1675
1676                 mtx_unlock(&softc->lock);
1677                 break;
1678         }
1679         default:
1680                 panic("invalid metatask type %d", metatask->tasktype);
1681                 break; /* NOTREACHED */
1682         }
1683 }
1684
1685 #ifdef oldapi
1686 void
1687 cfi_shutdown_shelf(cfi_cb_t callback, void *callback_arg)
1688 {
1689         struct ctl_mem_element *element;
1690         struct cfi_softc *softc;
1691         struct cfi_metatask *metatask;
1692
1693         softc = &fetd_internal_softc;
1694
1695         element = ctl_alloc_mem_element(&softc->metatask_pool, /*can_wait*/ 0);
1696         if (element == NULL) {
1697                 callback(callback_arg,
1698                          /*status*/ CFI_MT_ERROR,
1699                          /*sluns_found*/ 0,
1700                          /*sluns_complete*/ 0,
1701                          /*sluns_failed*/ 0);
1702                 return;
1703         }
1704
1705         metatask = (struct cfi_metatask *)element->bytes;
1706
1707         memset(metatask, 0, sizeof(*metatask));
1708         metatask->tasktype = CFI_TASK_SHUTDOWN;
1709         metatask->status = CFI_MT_NONE;
1710         metatask->taskinfo.startstop.callback = callback;
1711         metatask->taskinfo.startstop.callback_arg = callback_arg;
1712         metatask->element = element;
1713
1714         cfi_action(softc, metatask);
1715
1716         /*
1717          * - send a report luns to lun 0, get LUN list.
1718          * - send an inquiry to each lun
1719          * - send a stop/offline to each direct access LUN
1720          *    - if we get a reservation conflict, reset the LUN and then
1721          *      retry sending the stop/offline
1722          * - return status back to the caller
1723          */
1724 }
1725
1726 void
1727 cfi_start_shelf(cfi_cb_t callback, void *callback_arg)
1728 {
1729         struct ctl_mem_element *element;
1730         struct cfi_softc *softc;
1731         struct cfi_metatask *metatask;
1732
1733         softc = &fetd_internal_softc;
1734
1735         element = ctl_alloc_mem_element(&softc->metatask_pool, /*can_wait*/ 0);
1736         if (element == NULL) {
1737                 callback(callback_arg,
1738                          /*status*/ CFI_MT_ERROR,
1739                          /*sluns_found*/ 0,
1740                          /*sluns_complete*/ 0,
1741                          /*sluns_failed*/ 0);
1742                 return;
1743         }
1744
1745         metatask = (struct cfi_metatask *)element->bytes;
1746
1747         memset(metatask, 0, sizeof(*metatask));
1748         metatask->tasktype = CFI_TASK_STARTUP;
1749         metatask->status = CFI_MT_NONE;
1750         metatask->taskinfo.startstop.callback = callback;
1751         metatask->taskinfo.startstop.callback_arg = callback_arg;
1752         metatask->element = element;
1753
1754         cfi_action(softc, metatask);
1755
1756         /*
1757          * - send a report luns to lun 0, get LUN list.
1758          * - send an inquiry to each lun
1759          * - send a stop/offline to each direct access LUN
1760          *    - if we get a reservation conflict, reset the LUN and then
1761          *      retry sending the stop/offline
1762          * - return status back to the caller
1763          */
1764 }
1765
1766 #endif
1767
1768 struct cfi_metatask *
1769 cfi_alloc_metatask(int can_wait)
1770 {
1771         struct ctl_mem_element *element;
1772         struct cfi_metatask *metatask;
1773         struct cfi_softc *softc;
1774
1775         softc = &fetd_internal_softc;
1776
1777         element = ctl_alloc_mem_element(&softc->metatask_pool, can_wait);
1778         if (element == NULL)
1779                 return (NULL);
1780
1781         metatask = (struct cfi_metatask *)element->bytes;
1782         memset(metatask, 0, sizeof(*metatask));
1783         metatask->status = CFI_MT_NONE;
1784         metatask->element = element;
1785
1786         return (metatask);
1787 }
1788
1789 void
1790 cfi_free_metatask(struct cfi_metatask *metatask)
1791 {
1792         ctl_free_mem_element(metatask->element);
1793 }
1794
1795 /*
1796  * vim: ts=8
1797  */