sys/dev/virtio/scsi/virtio_scsi.c

   1 /*-
   2  * Copyright (c) 2012, Bryan Venteicher <bryanv@FreeBSD.org>
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  * 1. Redistributions of source code must retain the above copyright
   9  *    notice unmodified, this list of conditions, and the following
  10  *    disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 /* Driver for VirtIO SCSI devices. */
  28
  29 #include <sys/cdefs.h>
  30 __FBSDID("$FreeBSD$");
  31
  32 #include <sys/param.h>
  33 #include <sys/systm.h>
  34 #include <sys/kernel.h>
  35 #include <sys/kthread.h>
  36 #include <sys/malloc.h>
  37 #include <sys/module.h>
  38 #include <sys/sglist.h>
  39 #include <sys/sysctl.h>
  40 #include <sys/lock.h>
  41 #include <sys/mutex.h>
  42 #include <sys/callout.h>
  43 #include <sys/queue.h>
  44 #include <sys/sbuf.h>
  45
  46 #include <machine/stdarg.h>
  47
  48 #include <machine/bus.h>
  49 #include <machine/resource.h>
  50 #include <sys/bus.h>
  51 #include <sys/rman.h>
  52
  53 #include <cam/cam.h>
  54 #include <cam/cam_ccb.h>
  55 #include <cam/cam_sim.h>
  56 #include <cam/cam_periph.h>
  57 #include <cam/cam_xpt_sim.h>
  58 #include <cam/cam_debug.h>
  59 #include <cam/scsi/scsi_all.h>
  60 #include <cam/scsi/scsi_message.h>
  61
  62 #include <dev/virtio/virtio.h>
  63 #include <dev/virtio/virtqueue.h>
  64 #include <dev/virtio/scsi/virtio_scsi.h>
  65 #include <dev/virtio/scsi/virtio_scsivar.h>
  66
  67 #include "virtio_if.h"
  68
  69 static int      vtscsi_modevent(module_t, int, void *);
  70
  71 static int      vtscsi_probe(device_t);
  72 static int      vtscsi_attach(device_t);
  73 static int      vtscsi_detach(device_t);
  74 static int      vtscsi_suspend(device_t);
  75 static int      vtscsi_resume(device_t);
  76
  77 static void     vtscsi_negotiate_features(struct vtscsi_softc *);
  78 static int      vtscsi_maximum_segments(struct vtscsi_softc *, int);
  79 static int      vtscsi_alloc_virtqueues(struct vtscsi_softc *);
  80 static void     vtscsi_write_device_config(struct vtscsi_softc *);
  81 static int      vtscsi_reinit(struct vtscsi_softc *);
  82
  83 static int      vtscsi_alloc_cam(struct vtscsi_softc *);
  84 static int      vtscsi_register_cam(struct vtscsi_softc *);
  85 static void     vtscsi_free_cam(struct vtscsi_softc *);
  86 static void     vtscsi_cam_async(void *, uint32_t, struct cam_path *, void *);
  87 static int      vtscsi_register_async(struct vtscsi_softc *);
  88 static void     vtscsi_deregister_async(struct vtscsi_softc *);
  89 static void     vtscsi_cam_action(struct cam_sim *, union ccb *);
  90 static void     vtscsi_cam_poll(struct cam_sim *);
  91
  92 static void     vtscsi_cam_scsi_io(struct vtscsi_softc *, struct cam_sim *,
  93                     union ccb *);
  94 static void     vtscsi_cam_get_tran_settings(struct vtscsi_softc *,
  95                     union ccb *);
  96 static void     vtscsi_cam_reset_bus(struct vtscsi_softc *, union ccb *);
  97 static void     vtscsi_cam_reset_dev(struct vtscsi_softc *, union ccb *);
  98 static void     vtscsi_cam_abort(struct vtscsi_softc *, union ccb *);
  99 static void     vtscsi_cam_path_inquiry(struct vtscsi_softc *,
 100                     struct cam_sim *, union ccb *);
 101
 102 static int      vtscsi_sg_append_scsi_buf(struct vtscsi_softc *,
 103                     struct sglist *, struct ccb_scsiio *);
 104 static int      vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *,
 105                     struct vtscsi_request *, int *, int *);
 106 static int      vtscsi_execute_scsi_cmd(struct vtscsi_softc *,
 107                     struct vtscsi_request *);
 108 static int      vtscsi_start_scsi_cmd(struct vtscsi_softc *, union ccb *);
 109 static void     vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *,
 110                     struct vtscsi_request *);
 111 static int      vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *,
 112                     struct vtscsi_request *);
 113 static void     vtscsi_timedout_scsi_cmd(void *);
 114 static cam_status vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *);
 115 static cam_status vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *,
 116                     struct ccb_scsiio *, struct virtio_scsi_cmd_resp *);
 117 static void     vtscsi_complete_scsi_cmd(struct vtscsi_softc *,
 118                     struct vtscsi_request *);
 119
 120 static void     vtscsi_poll_ctrl_req(struct vtscsi_softc *,
 121                     struct vtscsi_request *);
 122 static int      vtscsi_execute_ctrl_req(struct vtscsi_softc *,
 123                     struct vtscsi_request *, struct sglist *, int, int, int);
 124 static void     vtscsi_complete_abort_task_cmd(struct vtscsi_softc *c,
 125                     struct vtscsi_request *);
 126 static int      vtscsi_execute_abort_task_cmd(struct vtscsi_softc *,
 127                     struct vtscsi_request *);
 128 static int      vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *,
 129                     struct vtscsi_request *);
 130
 131 static void     vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *);
 132 static void     vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []);
 133 static void     vtscsi_init_scsi_cmd_req(struct ccb_scsiio *,
 134                     struct virtio_scsi_cmd_req *);
 135 static void     vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t,
 136                     uintptr_t, struct virtio_scsi_ctrl_tmf_req *);
 137
 138 static void     vtscsi_freeze_simq(struct vtscsi_softc *, int);
 139 static int      vtscsi_thaw_simq(struct vtscsi_softc *, int);
 140
 141 static void     vtscsi_announce(struct vtscsi_softc *, uint32_t, target_id_t,
 142                     lun_id_t);
 143 static void     vtscsi_execute_rescan(struct vtscsi_softc *, target_id_t,
 144                     lun_id_t);
 145 static void     vtscsi_execute_rescan_bus(struct vtscsi_softc *);
 146
 147 static void     vtscsi_handle_event(struct vtscsi_softc *,
 148                     struct virtio_scsi_event *);
 149 static int      vtscsi_enqueue_event_buf(struct vtscsi_softc *,
 150                     struct virtio_scsi_event *);
 151 static int      vtscsi_init_event_vq(struct vtscsi_softc *);
 152 static void     vtscsi_reinit_event_vq(struct vtscsi_softc *);
 153 static void     vtscsi_drain_event_vq(struct vtscsi_softc *);
 154
 155 static void     vtscsi_complete_vqs_locked(struct vtscsi_softc *);
 156 static void     vtscsi_complete_vqs(struct vtscsi_softc *);
 157 static void     vtscsi_drain_vqs(struct vtscsi_softc *);
 158 static void     vtscsi_cancel_request(struct vtscsi_softc *,
 159                     struct vtscsi_request *);
 160 static void     vtscsi_drain_vq(struct vtscsi_softc *, struct virtqueue *);
 161 static void     vtscsi_stop(struct vtscsi_softc *);
 162 static int      vtscsi_reset_bus(struct vtscsi_softc *);
 163
 164 static void     vtscsi_init_request(struct vtscsi_softc *,
 165                     struct vtscsi_request *);
 166 static int      vtscsi_alloc_requests(struct vtscsi_softc *);
 167 static void     vtscsi_free_requests(struct vtscsi_softc *);
 168 static void     vtscsi_enqueue_request(struct vtscsi_softc *,
 169                     struct vtscsi_request *);
 170 static struct vtscsi_request * vtscsi_dequeue_request(struct vtscsi_softc *);
 171
 172 static void     vtscsi_complete_request(struct vtscsi_request *);
 173 static void     vtscsi_complete_vq(struct vtscsi_softc *, struct virtqueue *);
 174
 175 static void     vtscsi_control_vq_intr(void *);
 176 static void     vtscsi_event_vq_intr(void *);
 177 static void     vtscsi_request_vq_intr(void *);
 178 static void     vtscsi_disable_vqs_intr(struct vtscsi_softc *);
 179 static void     vtscsi_enable_vqs_intr(struct vtscsi_softc *);
 180
 181 static void     vtscsi_get_tunables(struct vtscsi_softc *);
 182 static void     vtscsi_add_sysctl(struct vtscsi_softc *);
 183
 184 static void     vtscsi_printf_req(struct vtscsi_request *, const char *,
 185                     const char *, ...);
 186
 187 /* Global tunables. */
 188 /*
 189  * The current QEMU VirtIO SCSI implementation does not cancel in-flight
 190  * IO during virtio_stop(). So in-flight requests still complete after the
 191  * device reset. We would have to wait for all the in-flight IO to complete,
 192  * which defeats the typical purpose of a bus reset. We could simulate the
 193  * bus reset with either I_T_NEXUS_RESET of all the targets, or with
 194  * LOGICAL_UNIT_RESET of all the LUNs (assuming there is space in the
 195  * control virtqueue). But this isn't very useful if things really go off
 196  * the rails, so default to disabled for now.
 197  */
 198 static int vtscsi_bus_reset_disable = 1;
 199 TUNABLE_INT("hw.vtscsi.bus_reset_disable", &vtscsi_bus_reset_disable);
 200
 201 static struct virtio_feature_desc vtscsi_feature_desc[] = {
 202         { VIRTIO_SCSI_F_INOUT,          "InOut"         },
 203         { VIRTIO_SCSI_F_HOTPLUG,        "Hotplug"       },
 204
 205         { 0, NULL }
 206 };
 207
 208 static device_method_t vtscsi_methods[] = {
 209         /* Device methods. */
 210         DEVMETHOD(device_probe,         vtscsi_probe),
 211         DEVMETHOD(device_attach,        vtscsi_attach),
 212         DEVMETHOD(device_detach,        vtscsi_detach),
 213         DEVMETHOD(device_suspend,       vtscsi_suspend),
 214         DEVMETHOD(device_resume,        vtscsi_resume),
 215
 216         DEVMETHOD_END
 217 };
 218
 219 static driver_t vtscsi_driver = {
 220         "vtscsi",
 221         vtscsi_methods,
 222         sizeof(struct vtscsi_softc)
 223 };
 224 static devclass_t vtscsi_devclass;
 225
 226 DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass,
 227     vtscsi_modevent, 0);
 228 MODULE_VERSION(virtio_scsi, 1);
 229 MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1);
 230 MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1);
 231
 232 static int
 233 vtscsi_modevent(module_t mod, int type, void *unused)
 234 {
 235         int error;
 236
 237         switch (type) {
 238         case MOD_LOAD:
 239         case MOD_QUIESCE:
 240         case MOD_UNLOAD:
 241         case MOD_SHUTDOWN:
 242                 error = 0;
 243                 break;
 244         default:
 245                 error = EOPNOTSUPP;
 246                 break;
 247         }
 248
 249         return (error);
 250 }
 251
 252 static int
 253 vtscsi_probe(device_t dev)
 254 {
 255
 256         if (virtio_get_device_type(dev) != VIRTIO_ID_SCSI)
 257                 return (ENXIO);
 258
 259         device_set_desc(dev, "VirtIO SCSI Adapter");
 260
 261         return (BUS_PROBE_DEFAULT);
 262 }
 263
 264 static int
 265 vtscsi_attach(device_t dev)
 266 {
 267         struct vtscsi_softc *sc;
 268         struct virtio_scsi_config scsicfg;
 269         int error;
 270
 271         sc = device_get_softc(dev);
 272         sc->vtscsi_dev = dev;
 273
 274         VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev));
 275         TAILQ_INIT(&sc->vtscsi_req_free);
 276
 277         vtscsi_get_tunables(sc);
 278         vtscsi_add_sysctl(sc);
 279
 280         virtio_set_feature_desc(dev, vtscsi_feature_desc);
 281         vtscsi_negotiate_features(sc);
 282
 283         if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
 284                 sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT;
 285         if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT))
 286                 sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL;
 287         if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG))
 288                 sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG;
 289
 290         virtio_read_device_config(dev, 0, &scsicfg,
 291             sizeof(struct virtio_scsi_config));
 292
 293         sc->vtscsi_max_channel = scsicfg.max_channel;
 294         sc->vtscsi_max_target = scsicfg.max_target;
 295         sc->vtscsi_max_lun = scsicfg.max_lun;
 296         sc->vtscsi_event_buf_size = scsicfg.event_info_size;
 297
 298         vtscsi_write_device_config(sc);
 299
 300         sc->vtscsi_max_nsegs = vtscsi_maximum_segments(sc, scsicfg.seg_max);
 301         sc->vtscsi_sglist = sglist_alloc(sc->vtscsi_max_nsegs, M_NOWAIT);
 302         if (sc->vtscsi_sglist == NULL) {
 303                 error = ENOMEM;
 304                 device_printf(dev, "cannot allocate sglist\n");
 305                 goto fail;
 306         }
 307
 308         error = vtscsi_alloc_virtqueues(sc);
 309         if (error) {
 310                 device_printf(dev, "cannot allocate virtqueues\n");
 311                 goto fail;
 312         }
 313
 314         error = vtscsi_init_event_vq(sc);
 315         if (error) {
 316                 device_printf(dev, "cannot populate the eventvq\n");
 317                 goto fail;
 318         }
 319
 320         error = vtscsi_alloc_requests(sc);
 321         if (error) {
 322                 device_printf(dev, "cannot allocate requests\n");
 323                 goto fail;
 324         }
 325
 326         error = vtscsi_alloc_cam(sc);
 327         if (error) {
 328                 device_printf(dev, "cannot allocate CAM structures\n");
 329                 goto fail;
 330         }
 331
 332         error = virtio_setup_intr(dev, INTR_TYPE_CAM);
 333         if (error) {
 334                 device_printf(dev, "cannot setup virtqueue interrupts\n");
 335                 goto fail;
 336         }
 337
 338         vtscsi_enable_vqs_intr(sc);
 339
 340         /*
 341          * Register with CAM after interrupts are enabled so we will get
 342          * notified of the probe responses.
 343          */
 344         error = vtscsi_register_cam(sc);
 345         if (error) {
 346                 device_printf(dev, "cannot register with CAM\n");
 347                 goto fail;
 348         }
 349
 350 fail:
 351         if (error)
 352                 vtscsi_detach(dev);
 353
 354         return (error);
 355 }
 356
 357 static int
 358 vtscsi_detach(device_t dev)
 359 {
 360         struct vtscsi_softc *sc;
 361
 362         sc = device_get_softc(dev);
 363
 364         VTSCSI_LOCK(sc);
 365         sc->vtscsi_flags |= VTSCSI_FLAG_DETACH;
 366         if (device_is_attached(dev))
 367                 vtscsi_stop(sc);
 368         VTSCSI_UNLOCK(sc);
 369
 370         vtscsi_complete_vqs(sc);
 371         vtscsi_drain_vqs(sc);
 372
 373         vtscsi_free_cam(sc);
 374         vtscsi_free_requests(sc);
 375
 376         if (sc->vtscsi_sglist != NULL) {
 377                 sglist_free(sc->vtscsi_sglist);
 378                 sc->vtscsi_sglist = NULL;
 379         }
 380
 381         VTSCSI_LOCK_DESTROY(sc);
 382
 383         return (0);
 384 }
 385
 386 static int
 387 vtscsi_suspend(device_t dev)
 388 {
 389
 390         return (0);
 391 }
 392
 393 static int
 394 vtscsi_resume(device_t dev)
 395 {
 396
 397         return (0);
 398 }
 399
 400 static void
 401 vtscsi_negotiate_features(struct vtscsi_softc *sc)
 402 {
 403         device_t dev;
 404         uint64_t features;
 405
 406         dev = sc->vtscsi_dev;
 407         features = virtio_negotiate_features(dev, VTSCSI_FEATURES);
 408         sc->vtscsi_features = features;
 409 }
 410
 411 static int
 412 vtscsi_maximum_segments(struct vtscsi_softc *sc, int seg_max)
 413 {
 414         int nsegs;
 415
 416         nsegs = VTSCSI_MIN_SEGMENTS;
 417
 418         if (seg_max > 0) {
 419                 nsegs += MIN(seg_max, MAXPHYS / PAGE_SIZE + 1);
 420                 if (sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT)
 421                         nsegs = MIN(nsegs, VIRTIO_MAX_INDIRECT);
 422         } else
 423                 nsegs += 1;
 424
 425         return (nsegs);
 426 }
 427
 428 static int
 429 vtscsi_alloc_virtqueues(struct vtscsi_softc *sc)
 430 {
 431         device_t dev;
 432         struct vq_alloc_info vq_info[3];
 433         int nvqs;
 434
 435         dev = sc->vtscsi_dev;
 436         nvqs = 3;
 437
 438         VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtscsi_control_vq_intr, sc,
 439             &sc->vtscsi_control_vq, "%s control", device_get_nameunit(dev));
 440
 441         VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtscsi_event_vq_intr, sc,
 442             &sc->vtscsi_event_vq, "%s event", device_get_nameunit(dev));
 443
 444         VQ_ALLOC_INFO_INIT(&vq_info[2], sc->vtscsi_max_nsegs,
 445             vtscsi_request_vq_intr, sc, &sc->vtscsi_request_vq,
 446             "%s request", device_get_nameunit(dev));
 447
 448         return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
 449 }
 450
 451 static void
 452 vtscsi_write_device_config(struct vtscsi_softc *sc)
 453 {
 454
 455         virtio_write_dev_config_4(sc->vtscsi_dev,
 456             offsetof(struct virtio_scsi_config, sense_size),
 457             VIRTIO_SCSI_SENSE_SIZE);
 458
 459         /*
 460          * This is the size in the virtio_scsi_cmd_req structure. Note
 461          * this value (32) is larger than the maximum CAM CDB size (16).
 462          */
 463         virtio_write_dev_config_4(sc->vtscsi_dev,
 464             offsetof(struct virtio_scsi_config, cdb_size),
 465             VIRTIO_SCSI_CDB_SIZE);
 466 }
 467
 468 static int
 469 vtscsi_reinit(struct vtscsi_softc *sc)
 470 {
 471         device_t dev;
 472         int error;
 473
 474         dev = sc->vtscsi_dev;
 475
 476         error = virtio_reinit(dev, sc->vtscsi_features);
 477         if (error == 0) {
 478                 vtscsi_write_device_config(sc);
 479                 vtscsi_reinit_event_vq(sc);
 480                 virtio_reinit_complete(dev);
 481
 482                 vtscsi_enable_vqs_intr(sc);
 483         }
 484
 485         vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d\n", error);
 486
 487         return (error);
 488 }
 489
 490 static int
 491 vtscsi_alloc_cam(struct vtscsi_softc *sc)
 492 {
 493         device_t dev;
 494         struct cam_devq *devq;
 495         int openings;
 496
 497         dev = sc->vtscsi_dev;
 498         openings = sc->vtscsi_nrequests - VTSCSI_RESERVED_REQUESTS;
 499
 500         devq = cam_simq_alloc(openings);
 501         if (devq == NULL) {
 502                 device_printf(dev, "cannot allocate SIM queue\n");
 503                 return (ENOMEM);
 504         }
 505
 506         sc->vtscsi_sim = cam_sim_alloc(vtscsi_cam_action, vtscsi_cam_poll,
 507             "vtscsi", sc, device_get_unit(dev), VTSCSI_MTX(sc), 1,
 508             openings, devq);
 509         if (sc->vtscsi_sim == NULL) {
 510                 cam_simq_free(devq);
 511                 device_printf(dev, "cannot allocate SIM\n");
 512                 return (ENOMEM);
 513         }
 514
 515         return (0);
 516 }
 517
 518 static int
 519 vtscsi_register_cam(struct vtscsi_softc *sc)
 520 {
 521         device_t dev;
 522         int registered, error;
 523
 524         dev = sc->vtscsi_dev;
 525         registered = 0;
 526
 527         VTSCSI_LOCK(sc);
 528
 529         if (xpt_bus_register(sc->vtscsi_sim, dev, 0) != CAM_SUCCESS) {
 530                 error = ENOMEM;
 531                 device_printf(dev, "cannot register XPT bus\n");
 532                 goto fail;
 533         }
 534
 535         registered = 1;
 536
 537         if (xpt_create_path(&sc->vtscsi_path, NULL,
 538             cam_sim_path(sc->vtscsi_sim), CAM_TARGET_WILDCARD,
 539             CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 540                 error = ENOMEM;
 541                 device_printf(dev, "cannot create bus path\n");
 542                 goto fail;
 543         }
 544
 545         if (vtscsi_register_async(sc) != CAM_REQ_CMP) {
 546                 error = EIO;
 547                 device_printf(dev, "cannot register async callback\n");
 548                 goto fail;
 549         }
 550
 551         VTSCSI_UNLOCK(sc);
 552
 553         return (0);
 554
 555 fail:
 556         if (sc->vtscsi_path != NULL) {
 557                 xpt_free_path(sc->vtscsi_path);
 558                 sc->vtscsi_path = NULL;
 559         }
 560
 561         if (registered != 0)
 562                 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim));
 563
 564         VTSCSI_UNLOCK(sc);
 565
 566         return (error);
 567 }
 568
 569 static void
 570 vtscsi_free_cam(struct vtscsi_softc *sc)
 571 {
 572
 573         VTSCSI_LOCK(sc);
 574
 575         if (sc->vtscsi_path != NULL) {
 576                 vtscsi_deregister_async(sc);
 577
 578                 xpt_free_path(sc->vtscsi_path);
 579                 sc->vtscsi_path = NULL;
 580
 581                 xpt_bus_deregister(cam_sim_path(sc->vtscsi_sim));
 582         }
 583
 584         if (sc->vtscsi_sim != NULL) {
 585                 cam_sim_free(sc->vtscsi_sim, 1);
 586                 sc->vtscsi_sim = NULL;
 587         }
 588
 589         VTSCSI_UNLOCK(sc);
 590 }
 591
 592 static void
 593 vtscsi_cam_async(void *cb_arg, uint32_t code, struct cam_path *path, void *arg)
 594 {
 595         struct cam_sim *sim;
 596         struct vtscsi_softc *sc;
 597
 598         sim = cb_arg;
 599         sc = cam_sim_softc(sim);
 600
 601         vtscsi_dprintf(sc, VTSCSI_TRACE, "code=%u\n", code);
 602
 603         /*
 604          * TODO Once QEMU supports event reporting, we should
 605          *      (un)subscribe to events here.
 606          */
 607         switch (code) {
 608         case AC_FOUND_DEVICE:
 609                 break;
 610         case AC_LOST_DEVICE:
 611                 break;
 612         }
 613 }
 614
 615 static int
 616 vtscsi_register_async(struct vtscsi_softc *sc)
 617 {
 618         struct ccb_setasync csa;
 619
 620         xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5);
 621         csa.ccb_h.func_code = XPT_SASYNC_CB;
 622         csa.event_enable = AC_LOST_DEVICE | AC_FOUND_DEVICE;
 623         csa.callback = vtscsi_cam_async;
 624         csa.callback_arg = sc->vtscsi_sim;
 625
 626         xpt_action((union ccb *) &csa);
 627
 628         return (csa.ccb_h.status);
 629 }
 630
 631 static void
 632 vtscsi_deregister_async(struct vtscsi_softc *sc)
 633 {
 634         struct ccb_setasync csa;
 635
 636         xpt_setup_ccb(&csa.ccb_h, sc->vtscsi_path, 5);
 637         csa.ccb_h.func_code = XPT_SASYNC_CB;
 638         csa.event_enable = 0;
 639         csa.callback = vtscsi_cam_async;
 640         csa.callback_arg = sc->vtscsi_sim;
 641
 642         xpt_action((union ccb *) &csa);
 643 }
 644
 645 static void
 646 vtscsi_cam_action(struct cam_sim *sim, union ccb *ccb)
 647 {
 648         struct vtscsi_softc *sc;
 649         struct ccb_hdr *ccbh;
 650
 651         sc = cam_sim_softc(sim);
 652         ccbh = &ccb->ccb_h;
 653
 654         VTSCSI_LOCK_OWNED(sc);
 655
 656         if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) {
 657                 /*
 658                  * The VTSCSI_MTX is briefly dropped between setting
 659                  * VTSCSI_FLAG_DETACH and deregistering with CAM, so
 660                  * drop any CCBs that come in during that window.
 661                  */
 662                 ccbh->status = CAM_NO_HBA;
 663                 xpt_done(ccb);
 664                 return;
 665         }
 666
 667         switch (ccbh->func_code) {
 668         case XPT_SCSI_IO:
 669                 vtscsi_cam_scsi_io(sc, sim, ccb);
 670                 break;
 671
 672         case XPT_SET_TRAN_SETTINGS:
 673                 ccbh->status = CAM_FUNC_NOTAVAIL;
 674                 xpt_done(ccb);
 675                 break;
 676
 677         case XPT_GET_TRAN_SETTINGS:
 678                 vtscsi_cam_get_tran_settings(sc, ccb);
 679                 break;
 680
 681         case XPT_RESET_BUS:
 682                 vtscsi_cam_reset_bus(sc, ccb);
 683                 break;
 684
 685         case XPT_RESET_DEV:
 686                 vtscsi_cam_reset_dev(sc, ccb);
 687                 break;
 688
 689         case XPT_ABORT:
 690                 vtscsi_cam_abort(sc, ccb);
 691                 break;
 692
 693         case XPT_CALC_GEOMETRY:
 694                 cam_calc_geometry(&ccb->ccg, 1);
 695                 xpt_done(ccb);
 696                 break;
 697
 698         case XPT_PATH_INQ:
 699                 vtscsi_cam_path_inquiry(sc, sim, ccb);
 700                 break;
 701
 702         default:
 703                 vtscsi_dprintf(sc, VTSCSI_ERROR,
 704                     "invalid ccb=%p func=%#x\n", ccb, ccbh->func_code);
 705
 706                 ccbh->status = CAM_REQ_INVALID;
 707                 xpt_done(ccb);
 708                 break;
 709         }
 710 }
 711
 712 static void
 713 vtscsi_cam_poll(struct cam_sim *sim)
 714 {
 715         struct vtscsi_softc *sc;
 716
 717         sc = cam_sim_softc(sim);
 718
 719         vtscsi_complete_vqs_locked(sc);
 720 }
 721
 722 static void
 723 vtscsi_cam_scsi_io(struct vtscsi_softc *sc, struct cam_sim *sim,
 724     union ccb *ccb)
 725 {
 726         struct ccb_hdr *ccbh;
 727         struct ccb_scsiio *csio;
 728         int error;
 729
 730         ccbh = &ccb->ccb_h;
 731         csio = &ccb->csio;
 732
 733         if (csio->cdb_len > VIRTIO_SCSI_CDB_SIZE) {
 734                 error = EINVAL;
 735                 ccbh->status = CAM_REQ_INVALID;
 736                 goto done;
 737         }
 738
 739         if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_BOTH &&
 740             (sc->vtscsi_flags & VTSCSI_FLAG_BIDIRECTIONAL) == 0) {
 741                 error = EINVAL;
 742                 ccbh->status = CAM_REQ_INVALID;
 743                 goto done;
 744         }
 745
 746         error = vtscsi_start_scsi_cmd(sc, ccb);
 747
 748 done:
 749         if (error) {
 750                 vtscsi_dprintf(sc, VTSCSI_ERROR,
 751                     "error=%d ccb=%p status=%#x\n", error, ccb, ccbh->status);
 752                 xpt_done(ccb);
 753         }
 754 }
 755
 756 static void
 757 vtscsi_cam_get_tran_settings(struct vtscsi_softc *sc, union ccb *ccb)
 758 {
 759         struct ccb_trans_settings *cts;
 760         struct ccb_trans_settings_scsi *scsi;
 761
 762         cts = &ccb->cts;
 763         scsi = &cts->proto_specific.scsi;
 764
 765         cts->protocol = PROTO_SCSI;
 766         cts->protocol_version = SCSI_REV_SPC3;
 767         cts->transport = XPORT_SAS;
 768         cts->transport_version = 0;
 769
 770         scsi->valid = CTS_SCSI_VALID_TQ;
 771         scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
 772
 773         ccb->ccb_h.status = CAM_REQ_CMP;
 774         xpt_done(ccb);
 775 }
 776
 777 static void
 778 vtscsi_cam_reset_bus(struct vtscsi_softc *sc, union ccb *ccb)
 779 {
 780         int error;
 781
 782         error = vtscsi_reset_bus(sc);
 783         if (error == 0)
 784                 ccb->ccb_h.status = CAM_REQ_CMP;
 785         else
 786                 ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 787
 788         vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d ccb=%p status=%#x\n",
 789             error, ccb, ccb->ccb_h.status);
 790
 791         xpt_done(ccb);
 792 }
 793
 794 static void
 795 vtscsi_cam_reset_dev(struct vtscsi_softc *sc, union ccb *ccb)
 796 {
 797         struct ccb_hdr *ccbh;
 798         struct vtscsi_request *req;
 799         int error;
 800
 801         ccbh = &ccb->ccb_h;
 802
 803         req = vtscsi_dequeue_request(sc);
 804         if (req == NULL) {
 805                 error = EAGAIN;
 806                 vtscsi_freeze_simq(sc, VTSCSI_REQUEST);
 807                 goto fail;
 808         }
 809
 810         req->vsr_ccb = ccb;
 811
 812         error = vtscsi_execute_reset_dev_cmd(sc, req);
 813         if (error == 0)
 814                 return;
 815
 816         vtscsi_enqueue_request(sc, req);
 817
 818 fail:
 819         vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n",
 820             error, req, ccb);
 821
 822         if (error == EAGAIN)
 823                 ccbh->status = CAM_RESRC_UNAVAIL;
 824         else
 825                 ccbh->status = CAM_REQ_CMP_ERR;
 826
 827         xpt_done(ccb);
 828 }
 829
 830 static void
 831 vtscsi_cam_abort(struct vtscsi_softc *sc, union ccb *ccb)
 832 {
 833         struct vtscsi_request *req;
 834         struct ccb_hdr *ccbh;
 835         int error;
 836
 837         ccbh = &ccb->ccb_h;
 838
 839         req = vtscsi_dequeue_request(sc);
 840         if (req == NULL) {
 841                 error = EAGAIN;
 842                 vtscsi_freeze_simq(sc, VTSCSI_REQUEST);
 843                 goto fail;
 844         }
 845
 846         req->vsr_ccb = ccb;
 847
 848         error = vtscsi_execute_abort_task_cmd(sc, req);
 849         if (error == 0)
 850                 return;
 851
 852         vtscsi_enqueue_request(sc, req);
 853
 854 fail:
 855         vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p\n",
 856             error, req, ccb);
 857
 858         if (error == EAGAIN)
 859                 ccbh->status = CAM_RESRC_UNAVAIL;
 860         else
 861                 ccbh->status = CAM_REQ_CMP_ERR;
 862
 863         xpt_done(ccb);
 864 }
 865
 866 static void
 867 vtscsi_cam_path_inquiry(struct vtscsi_softc *sc, struct cam_sim *sim,
 868     union ccb *ccb)
 869 {
 870         device_t dev;
 871         struct ccb_pathinq *cpi;
 872
 873         dev = sc->vtscsi_dev;
 874         cpi = &ccb->cpi;
 875
 876         vtscsi_dprintf(sc, VTSCSI_TRACE, "sim=%p ccb=%p\n", sim, ccb);
 877
 878         cpi->version_num = 1;
 879         cpi->hba_inquiry = PI_TAG_ABLE;
 880         cpi->target_sprt = 0;
 881         cpi->hba_misc = PIM_SEQSCAN;
 882         if (vtscsi_bus_reset_disable != 0)
 883                 cpi->hba_misc |= PIM_NOBUSRESET;
 884         cpi->hba_eng_cnt = 0;
 885
 886         cpi->max_target = sc->vtscsi_max_target;
 887         cpi->max_lun = sc->vtscsi_max_lun;
 888         cpi->initiator_id = VTSCSI_INITIATOR_ID;
 889
 890         strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 891         strncpy(cpi->hba_vid, "VirtIO", HBA_IDLEN);
 892         strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 893
 894         cpi->unit_number = cam_sim_unit(sim);
 895         cpi->bus_id = cam_sim_bus(sim);
 896
 897         cpi->base_transfer_speed = 300000;
 898
 899         cpi->protocol = PROTO_SCSI;
 900         cpi->protocol_version = SCSI_REV_SPC3;
 901         cpi->transport = XPORT_SAS;
 902         cpi->transport_version = 0;
 903
 904         cpi->maxio = (sc->vtscsi_max_nsegs - VTSCSI_MIN_SEGMENTS - 1) *
 905             PAGE_SIZE;
 906
 907         cpi->hba_vendor = virtio_get_vendor(dev);
 908         cpi->hba_device = virtio_get_device(dev);
 909         cpi->hba_subvendor = virtio_get_subvendor(dev);
 910         cpi->hba_subdevice = virtio_get_subdevice(dev);
 911
 912         ccb->ccb_h.status = CAM_REQ_CMP;
 913         xpt_done(ccb);
 914 }
 915
 916 static int
 917 vtscsi_sg_append_scsi_buf(struct vtscsi_softc *sc, struct sglist *sg,
 918     struct ccb_scsiio *csio)
 919 {
 920         struct ccb_hdr *ccbh;
 921         struct bus_dma_segment *dseg;
 922         int i, error;
 923
 924         ccbh = &csio->ccb_h;
 925         error = 0;
 926
 927         switch ((ccbh->flags & CAM_DATA_MASK)) {
 928         case CAM_DATA_VADDR:
 929                 error = sglist_append(sg, csio->data_ptr, csio->dxfer_len);
 930                 break;
 931         case CAM_DATA_PADDR:
 932                 error = sglist_append_phys(sg,
 933                     (vm_paddr_t)(vm_offset_t) csio->data_ptr, csio->dxfer_len);
 934                 break;
 935         case CAM_DATA_SG:
 936                 for (i = 0; i < csio->sglist_cnt && error == 0; i++) {
 937                         dseg = &((struct bus_dma_segment *)csio->data_ptr)[i];
 938                         error = sglist_append(sg,
 939                             (void *)(vm_offset_t) dseg->ds_addr, dseg->ds_len);
 940                 }
 941                 break;
 942         case CAM_DATA_SG_PADDR:
 943                 for (i = 0; i < csio->sglist_cnt && error == 0; i++) {
 944                         dseg = &((struct bus_dma_segment *)csio->data_ptr)[i];
 945                         error = sglist_append_phys(sg,
 946                             (vm_paddr_t) dseg->ds_addr, dseg->ds_len);
 947                 }
 948                 break;
 949         default:
 950                 error = EINVAL;
 951                 break;
 952         }
 953
 954         return (error);
 955 }
 956
 957 static int
 958 vtscsi_fill_scsi_cmd_sglist(struct vtscsi_softc *sc, struct vtscsi_request *req,
 959     int *readable, int *writable)
 960 {
 961         struct sglist *sg;
 962         struct ccb_hdr *ccbh;
 963         struct ccb_scsiio *csio;
 964         struct virtio_scsi_cmd_req *cmd_req;
 965         struct virtio_scsi_cmd_resp *cmd_resp;
 966         int error;
 967
 968         sg = sc->vtscsi_sglist;
 969         csio = &req->vsr_ccb->csio;
 970         ccbh = &csio->ccb_h;
 971         cmd_req = &req->vsr_cmd_req;
 972         cmd_resp = &req->vsr_cmd_resp;
 973
 974         sglist_reset(sg);
 975
 976         sglist_append(sg, cmd_req, sizeof(struct virtio_scsi_cmd_req));
 977         if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
 978                 error = vtscsi_sg_append_scsi_buf(sc, sg, csio);
 979                 /* At least one segment must be left for the response. */
 980                 if (error || sg->sg_nseg == sg->sg_maxseg)
 981                         goto fail;
 982         }
 983
 984         *readable = sg->sg_nseg;
 985
 986         sglist_append(sg, cmd_resp, sizeof(struct virtio_scsi_cmd_resp));
 987         if ((ccbh->flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 988                 error = vtscsi_sg_append_scsi_buf(sc, sg, csio);
 989                 if (error)
 990                         goto fail;
 991         }
 992
 993         *writable = sg->sg_nseg - *readable;
 994
 995         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p readable=%d "
 996             "writable=%d\n", req, ccbh, *readable, *writable);
 997
 998         return (0);
 999
1000 fail:
1001         /*
1002          * This should never happen unless maxio was incorrectly set.
1003          */
1004         vtscsi_set_ccb_status(ccbh, CAM_REQ_TOO_BIG, 0);
1005
1006         vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p ccb=%p "
1007             "nseg=%d maxseg=%d\n",
1008             error, req, ccbh, sg->sg_nseg, sg->sg_maxseg);
1009
1010         return (EFBIG);
1011 }
1012
1013 static int
1014 vtscsi_execute_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req)
1015 {
1016         struct sglist *sg;
1017         struct virtqueue *vq;
1018         struct ccb_scsiio *csio;
1019         struct ccb_hdr *ccbh;
1020         struct virtio_scsi_cmd_req *cmd_req;
1021         struct virtio_scsi_cmd_resp *cmd_resp;
1022         int readable, writable, error;
1023
1024         sg = sc->vtscsi_sglist;
1025         vq = sc->vtscsi_request_vq;
1026         csio = &req->vsr_ccb->csio;
1027         ccbh = &csio->ccb_h;
1028         cmd_req = &req->vsr_cmd_req;
1029         cmd_resp = &req->vsr_cmd_resp;
1030
1031         vtscsi_init_scsi_cmd_req(csio, cmd_req);
1032
1033         error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable);
1034         if (error)
1035                 return (error);
1036
1037         req->vsr_complete = vtscsi_complete_scsi_cmd;
1038         cmd_resp->response = -1;
1039
1040         error = virtqueue_enqueue(vq, req, sg, readable, writable);
1041         if (error) {
1042                 vtscsi_dprintf(sc, VTSCSI_ERROR,
1043                     "enqueue error=%d req=%p ccb=%p\n", error, req, ccbh);
1044
1045                 ccbh->status = CAM_REQUEUE_REQ;
1046                 vtscsi_freeze_simq(sc, VTSCSI_REQUEST_VQ);
1047                 return (error);
1048         }
1049
1050         ccbh->status |= CAM_SIM_QUEUED;
1051         ccbh->ccbh_vtscsi_req = req;
1052
1053         virtqueue_notify(vq);
1054
1055         if (ccbh->timeout != CAM_TIME_INFINITY) {
1056                 req->vsr_flags |= VTSCSI_REQ_FLAG_TIMEOUT_SET;
1057                 callout_reset(&req->vsr_callout, ccbh->timeout * hz / 1000,
1058                     vtscsi_timedout_scsi_cmd, req);
1059         }
1060
1061         vtscsi_dprintf_req(req, VTSCSI_TRACE, "enqueued req=%p ccb=%p\n",
1062             req, ccbh);
1063
1064         return (0);
1065 }
1066
1067 static int
1068 vtscsi_start_scsi_cmd(struct vtscsi_softc *sc, union ccb *ccb)
1069 {
1070         struct vtscsi_request *req;
1071         int error;
1072
1073         req = vtscsi_dequeue_request(sc);
1074         if (req == NULL) {
1075                 ccb->ccb_h.status = CAM_REQUEUE_REQ;
1076                 vtscsi_freeze_simq(sc, VTSCSI_REQUEST);
1077                 return (ENOBUFS);
1078         }
1079
1080         req->vsr_ccb = ccb;
1081
1082         error = vtscsi_execute_scsi_cmd(sc, req);
1083         if (error)
1084                 vtscsi_enqueue_request(sc, req);
1085
1086         return (error);
1087 }
1088
1089 static void
1090 vtscsi_complete_abort_timedout_scsi_cmd(struct vtscsi_softc *sc,
1091     struct vtscsi_request *req)
1092 {
1093         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1094         struct vtscsi_request *to_req;
1095         uint8_t response;
1096
1097         tmf_resp = &req->vsr_tmf_resp;
1098         response = tmf_resp->response;
1099         to_req = req->vsr_timedout_req;
1100
1101         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p to_req=%p response=%d\n",
1102             req, to_req, response);
1103
1104         vtscsi_enqueue_request(sc, req);
1105
1106         /*
1107          * The timedout request could have completed between when the
1108          * abort task was sent and when the host processed it.
1109          */
1110         if (to_req->vsr_state != VTSCSI_REQ_STATE_TIMEDOUT)
1111                 return;
1112
1113         /* The timedout request was successfully aborted. */
1114         if (response == VIRTIO_SCSI_S_FUNCTION_COMPLETE)
1115                 return;
1116
1117         /* Don't bother if the device is going away. */
1118         if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH)
1119                 return;
1120
1121         /* The timedout request will be aborted by the reset. */
1122         if (sc->vtscsi_flags & VTSCSI_FLAG_RESET)
1123                 return;
1124
1125         vtscsi_reset_bus(sc);
1126 }
1127
1128 static int
1129 vtscsi_abort_timedout_scsi_cmd(struct vtscsi_softc *sc,
1130     struct vtscsi_request *to_req)
1131 {
1132         struct sglist *sg;
1133         struct ccb_hdr *to_ccbh;
1134         struct vtscsi_request *req;
1135         struct virtio_scsi_ctrl_tmf_req *tmf_req;
1136         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1137         int error;
1138
1139         sg = sc->vtscsi_sglist;
1140         to_ccbh = &to_req->vsr_ccb->ccb_h;
1141
1142         req = vtscsi_dequeue_request(sc);
1143         if (req == NULL) {
1144                 error = ENOBUFS;
1145                 goto fail;
1146         }
1147
1148         tmf_req = &req->vsr_tmf_req;
1149         tmf_resp = &req->vsr_tmf_resp;
1150
1151         vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
1152             (uintptr_t) to_ccbh, tmf_req);
1153
1154         sglist_reset(sg);
1155         sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));
1156         sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp));
1157
1158         req->vsr_timedout_req = to_req;
1159         req->vsr_complete = vtscsi_complete_abort_timedout_scsi_cmd;
1160         tmf_resp->response = -1;
1161
1162         error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1,
1163             VTSCSI_EXECUTE_ASYNC);
1164         if (error == 0)
1165                 return (0);
1166
1167         vtscsi_enqueue_request(sc, req);
1168
1169 fail:
1170         vtscsi_dprintf(sc, VTSCSI_ERROR, "error=%d req=%p "
1171             "timedout req=%p ccb=%p\n", error, req, to_req, to_ccbh);
1172
1173         return (error);
1174 }
1175
1176 static void
1177 vtscsi_timedout_scsi_cmd(void *xreq)
1178 {
1179         struct vtscsi_softc *sc;
1180         struct vtscsi_request *to_req;
1181
1182         to_req = xreq;
1183         sc = to_req->vsr_softc;
1184
1185         vtscsi_dprintf(sc, VTSCSI_INFO, "timedout req=%p ccb=%p state=%#x\n",
1186             to_req, to_req->vsr_ccb, to_req->vsr_state);
1187
1188         /* Don't bother if the device is going away. */
1189         if (sc->vtscsi_flags & VTSCSI_FLAG_DETACH)
1190                 return;
1191
1192         /*
1193          * Bail if the request is not in use. We likely raced when
1194          * stopping the callout handler or it has already been aborted.
1195          */
1196         if (to_req->vsr_state != VTSCSI_REQ_STATE_INUSE ||
1197             (to_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) == 0)
1198                 return;
1199
1200         /*
1201          * Complete the request queue in case the timedout request is
1202          * actually just pending.
1203          */
1204         vtscsi_complete_vq(sc, sc->vtscsi_request_vq);
1205         if (to_req->vsr_state == VTSCSI_REQ_STATE_FREE)
1206                 return;
1207
1208         sc->vtscsi_stats.scsi_cmd_timeouts++;
1209         to_req->vsr_state = VTSCSI_REQ_STATE_TIMEDOUT;
1210
1211         if (vtscsi_abort_timedout_scsi_cmd(sc, to_req) == 0)
1212                 return;
1213
1214         vtscsi_dprintf(sc, VTSCSI_ERROR, "resetting bus\n");
1215         vtscsi_reset_bus(sc);
1216 }
1217
1218 static cam_status
1219 vtscsi_scsi_cmd_cam_status(struct virtio_scsi_cmd_resp *cmd_resp)
1220 {
1221         cam_status status;
1222
1223         switch (cmd_resp->response) {
1224         case VIRTIO_SCSI_S_OK:
1225                 status = CAM_REQ_CMP;
1226                 break;
1227         case VIRTIO_SCSI_S_OVERRUN:
1228                 status = CAM_DATA_RUN_ERR;
1229                 break;
1230         case VIRTIO_SCSI_S_ABORTED:
1231                 status = CAM_REQ_ABORTED;
1232                 break;
1233         case VIRTIO_SCSI_S_BAD_TARGET:
1234                 status = CAM_SEL_TIMEOUT;
1235                 break;
1236         case VIRTIO_SCSI_S_RESET:
1237                 status = CAM_SCSI_BUS_RESET;
1238                 break;
1239         case VIRTIO_SCSI_S_BUSY:
1240                 status = CAM_SCSI_BUSY;
1241                 break;
1242         case VIRTIO_SCSI_S_TRANSPORT_FAILURE:
1243         case VIRTIO_SCSI_S_TARGET_FAILURE:
1244         case VIRTIO_SCSI_S_NEXUS_FAILURE:
1245                 status = CAM_SCSI_IT_NEXUS_LOST;
1246                 break;
1247         default: /* VIRTIO_SCSI_S_FAILURE */
1248                 status = CAM_REQ_CMP_ERR;
1249                 break;
1250         }
1251
1252         return (status);
1253 }
1254
1255 static cam_status
1256 vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc,
1257     struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp)
1258 {
1259         cam_status status;
1260
1261         csio->scsi_status = cmd_resp->status;
1262         csio->resid = cmd_resp->resid;
1263
1264         if (csio->scsi_status == SCSI_STATUS_OK)
1265                 status = CAM_REQ_CMP;
1266         else
1267                 status = CAM_SCSI_STATUS_ERROR;
1268
1269         if (cmd_resp->sense_len > 0) {
1270                 status |= CAM_AUTOSNS_VALID;
1271
1272                 if (cmd_resp->sense_len < csio->sense_len)
1273                         csio->sense_resid = csio->sense_len -
1274                             cmd_resp->sense_len;
1275                 else
1276                         csio->sense_resid = 0;
1277
1278                 bzero(&csio->sense_data, sizeof(csio->sense_data));
1279                 memcpy(cmd_resp->sense, &csio->sense_data,
1280                     csio->sense_len - csio->sense_resid);
1281         }
1282
1283         vtscsi_dprintf(sc, status == CAM_REQ_CMP ? VTSCSI_TRACE : VTSCSI_ERROR,
1284             "ccb=%p scsi_status=%#x resid=%u sense_resid=%u\n",
1285             csio, csio->scsi_status, csio->resid, csio->sense_resid);
1286
1287         return (status);
1288 }
1289
1290 static void
1291 vtscsi_complete_scsi_cmd(struct vtscsi_softc *sc, struct vtscsi_request *req)
1292 {
1293         struct ccb_hdr *ccbh;
1294         struct ccb_scsiio *csio;
1295         struct virtio_scsi_cmd_resp *cmd_resp;
1296         cam_status status;
1297
1298         csio = &req->vsr_ccb->csio;
1299         ccbh = &csio->ccb_h;
1300         cmd_resp = &req->vsr_cmd_resp;
1301
1302         KASSERT(ccbh->ccbh_vtscsi_req == req,
1303             ("ccb %p req mismatch %p/%p", ccbh, ccbh->ccbh_vtscsi_req, req));
1304
1305         if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET)
1306                 callout_stop(&req->vsr_callout);
1307
1308         status = vtscsi_scsi_cmd_cam_status(cmd_resp);
1309         if (status == CAM_REQ_ABORTED) {
1310                 if (req->vsr_state == VTSCSI_REQ_STATE_TIMEDOUT)
1311                         status = CAM_CMD_TIMEOUT;
1312         } else if (status == CAM_REQ_CMP)
1313                 status = vtscsi_complete_scsi_cmd_response(sc, csio, cmd_resp);
1314
1315         if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
1316                 status |= CAM_DEV_QFRZN;
1317                 xpt_freeze_devq(ccbh->path, 1);
1318         }
1319
1320         if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0)
1321                 status |= CAM_RELEASE_SIMQ;
1322
1323         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p status=%#x\n",
1324             req, ccbh, status);
1325
1326         ccbh->status = status;
1327         xpt_done(req->vsr_ccb);
1328         vtscsi_enqueue_request(sc, req);
1329 }
1330
1331 static void
1332 vtscsi_poll_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req)
1333 {
1334
1335         /* XXX We probably shouldn't poll forever. */
1336         req->vsr_flags |= VTSCSI_REQ_FLAG_POLLED;
1337         do
1338                 vtscsi_complete_vq(sc, sc->vtscsi_control_vq);
1339         while ((req->vsr_flags & VTSCSI_REQ_FLAG_COMPLETE) == 0);
1340
1341         req->vsr_flags &= ~VTSCSI_REQ_FLAG_POLLED;
1342 }
1343
1344 static int
1345 vtscsi_execute_ctrl_req(struct vtscsi_softc *sc, struct vtscsi_request *req,
1346     struct sglist *sg, int readable, int writable, int flag)
1347 {
1348         struct virtqueue *vq;
1349         int error;
1350
1351         vq = sc->vtscsi_control_vq;
1352
1353         MPASS(flag == VTSCSI_EXECUTE_POLL || req->vsr_complete != NULL);
1354
1355         error = virtqueue_enqueue(vq, req, sg, readable, writable);
1356         if (error) {
1357                 /*
1358                  * Return EAGAIN when the virtqueue does not have enough
1359                  * descriptors available.
1360                  */
1361                 if (error == ENOSPC || error == EMSGSIZE)
1362                         error = EAGAIN;
1363
1364                 return (error);
1365         }
1366
1367         virtqueue_notify(vq);
1368         if (flag == VTSCSI_EXECUTE_POLL)
1369                 vtscsi_poll_ctrl_req(sc, req);
1370
1371         return (0);
1372 }
1373
1374 static void
1375 vtscsi_complete_abort_task_cmd(struct vtscsi_softc *sc,
1376     struct vtscsi_request *req)
1377 {
1378         union ccb *ccb;
1379         struct ccb_hdr *ccbh;
1380         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1381
1382         ccb = req->vsr_ccb;
1383         ccbh = &ccb->ccb_h;
1384         tmf_resp = &req->vsr_tmf_resp;
1385
1386         switch (tmf_resp->response) {
1387         case VIRTIO_SCSI_S_FUNCTION_COMPLETE:
1388                 ccbh->status = CAM_REQ_CMP;
1389                 break;
1390         case VIRTIO_SCSI_S_FUNCTION_REJECTED:
1391                 ccbh->status = CAM_UA_ABORT;
1392                 break;
1393         default:
1394                 ccbh->status = CAM_REQ_CMP_ERR;
1395                 break;
1396         }
1397
1398         xpt_done(ccb);
1399         vtscsi_enqueue_request(sc, req);
1400 }
1401
1402 static int
1403 vtscsi_execute_abort_task_cmd(struct vtscsi_softc *sc,
1404     struct vtscsi_request *req)
1405 {
1406         struct sglist *sg;
1407         struct ccb_abort *cab;
1408         struct ccb_hdr *ccbh;
1409         struct ccb_hdr *abort_ccbh;
1410         struct vtscsi_request *abort_req;
1411         struct virtio_scsi_ctrl_tmf_req *tmf_req;
1412         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1413         int error;
1414
1415         sg = sc->vtscsi_sglist;
1416         cab = &req->vsr_ccb->cab;
1417         ccbh = &cab->ccb_h;
1418         tmf_req = &req->vsr_tmf_req;
1419         tmf_resp = &req->vsr_tmf_resp;
1420
1421         /* CCB header and request that's to be aborted. */
1422         abort_ccbh = &cab->abort_ccb->ccb_h;
1423         abort_req = abort_ccbh->ccbh_vtscsi_req;
1424
1425         if (abort_ccbh->func_code != XPT_SCSI_IO || abort_req == NULL) {
1426                 error = EINVAL;
1427                 goto fail;
1428         }
1429
1430         /* Only attempt to abort requests that could be in-flight. */
1431         if (abort_req->vsr_state != VTSCSI_REQ_STATE_INUSE) {
1432                 error = EALREADY;
1433                 goto fail;
1434         }
1435
1436         abort_req->vsr_state = VTSCSI_REQ_STATE_ABORTED;
1437         if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET)
1438                 callout_stop(&abort_req->vsr_callout);
1439
1440         vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
1441             (uintptr_t) abort_ccbh, tmf_req);
1442
1443         sglist_reset(sg);
1444         sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));
1445         sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp));
1446
1447         req->vsr_complete = vtscsi_complete_abort_task_cmd;
1448         tmf_resp->response = -1;
1449
1450         error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1,
1451             VTSCSI_EXECUTE_ASYNC);
1452
1453 fail:
1454         vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p abort_ccb=%p "
1455             "abort_req=%p\n", error, req, abort_ccbh, abort_req);
1456
1457         return (error);
1458 }
1459
1460 static void
1461 vtscsi_complete_reset_dev_cmd(struct vtscsi_softc *sc,
1462     struct vtscsi_request *req)
1463 {
1464         union ccb *ccb;
1465         struct ccb_hdr *ccbh;
1466         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1467
1468         ccb = req->vsr_ccb;
1469         ccbh = &ccb->ccb_h;
1470         tmf_resp = &req->vsr_tmf_resp;
1471
1472         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p response=%d\n",
1473             req, ccb, tmf_resp->response);
1474
1475         if (tmf_resp->response == VIRTIO_SCSI_S_FUNCTION_COMPLETE) {
1476                 ccbh->status = CAM_REQ_CMP;
1477                 vtscsi_announce(sc, AC_SENT_BDR, ccbh->target_id,
1478                     ccbh->target_lun);
1479         } else
1480                 ccbh->status = CAM_REQ_CMP_ERR;
1481
1482         xpt_done(ccb);
1483         vtscsi_enqueue_request(sc, req);
1484 }
1485
1486 static int
1487 vtscsi_execute_reset_dev_cmd(struct vtscsi_softc *sc,
1488     struct vtscsi_request *req)
1489 {
1490         struct sglist *sg;
1491         struct ccb_resetdev *crd;
1492         struct ccb_hdr *ccbh;
1493         struct virtio_scsi_ctrl_tmf_req *tmf_req;
1494         struct virtio_scsi_ctrl_tmf_resp *tmf_resp;
1495         uint32_t subtype;
1496         int error;
1497
1498         sg = sc->vtscsi_sglist;
1499         crd = &req->vsr_ccb->crd;
1500         ccbh = &crd->ccb_h;
1501         tmf_req = &req->vsr_tmf_req;
1502         tmf_resp = &req->vsr_tmf_resp;
1503
1504         if (ccbh->target_lun == CAM_LUN_WILDCARD)
1505                 subtype = VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET;
1506         else
1507                 subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET;
1508
1509         vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req);
1510
1511         sglist_reset(sg);
1512         sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));
1513         sglist_append(sg, tmf_resp, sizeof(struct virtio_scsi_ctrl_tmf_resp));
1514
1515         req->vsr_complete = vtscsi_complete_reset_dev_cmd;
1516         tmf_resp->response = -1;
1517
1518         error = vtscsi_execute_ctrl_req(sc, req, sg, 1, 1,
1519             VTSCSI_EXECUTE_ASYNC);
1520
1521         vtscsi_dprintf(sc, VTSCSI_TRACE, "error=%d req=%p ccb=%p\n",
1522             error, req, ccbh);
1523
1524         return (error);
1525 }
1526
1527 static void
1528 vtscsi_get_request_lun(uint8_t lun[], target_id_t *target_id, lun_id_t *lun_id)
1529 {
1530
1531         *target_id = lun[1];
1532         *lun_id = (lun[2] << 8) | lun[3];
1533 }
1534
1535 static void
1536 vtscsi_set_request_lun(struct ccb_hdr *ccbh, uint8_t lun[])
1537 {
1538
1539         lun[0] = 1;
1540         lun[1] = ccbh->target_id;
1541         lun[2] = 0x40 | ((ccbh->target_lun >> 8) & 0x3F);
1542         lun[3] = (ccbh->target_lun >> 8) & 0xFF;
1543 }
1544
1545 static void
1546 vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio,
1547     struct virtio_scsi_cmd_req *cmd_req)
1548 {
1549         uint8_t attr;
1550
1551         switch (csio->tag_action) {
1552         case MSG_HEAD_OF_Q_TAG:
1553                 attr = VIRTIO_SCSI_S_HEAD;
1554                 break;
1555         case MSG_ORDERED_Q_TAG:
1556                 attr = VIRTIO_SCSI_S_ORDERED;
1557                 break;
1558         case MSG_ACA_TASK:
1559                 attr = VIRTIO_SCSI_S_ACA;
1560                 break;
1561         default: /* MSG_SIMPLE_Q_TAG */
1562                 attr = VIRTIO_SCSI_S_SIMPLE;
1563                 break;
1564         }
1565
1566         vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun);
1567         cmd_req->tag = (uintptr_t) csio;
1568         cmd_req->task_attr = attr;
1569
1570         memcpy(cmd_req->cdb,
1571             csio->ccb_h.flags & CAM_CDB_POINTER ?
1572                 csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes,
1573             csio->cdb_len);
1574 }
1575
1576 static void
1577 vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype,
1578     uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req)
1579 {
1580
1581         vtscsi_set_request_lun(ccbh, tmf_req->lun);
1582
1583         tmf_req->type = VIRTIO_SCSI_T_TMF;
1584         tmf_req->subtype = subtype;
1585         tmf_req->tag = tag;
1586 }
1587
1588 static void
1589 vtscsi_freeze_simq(struct vtscsi_softc *sc, int reason)
1590 {
1591         int frozen;
1592
1593         frozen = sc->vtscsi_frozen;
1594
1595         if (reason & VTSCSI_REQUEST &&
1596             (sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS) == 0)
1597                 sc->vtscsi_frozen |= VTSCSI_FROZEN_NO_REQUESTS;
1598
1599         if (reason & VTSCSI_REQUEST_VQ &&
1600             (sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL) == 0)
1601                 sc->vtscsi_frozen |= VTSCSI_FROZEN_REQUEST_VQ_FULL;
1602
1603         /* Freeze the SIMQ if transitioned to frozen. */
1604         if (frozen == 0 && sc->vtscsi_frozen != 0) {
1605                 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ frozen\n");
1606                 xpt_freeze_simq(sc->vtscsi_sim, 1);
1607         }
1608 }
1609
1610 static int
1611 vtscsi_thaw_simq(struct vtscsi_softc *sc, int reason)
1612 {
1613         int thawed;
1614
1615         if (sc->vtscsi_frozen == 0 || reason == 0)
1616                 return (0);
1617
1618         if (reason & VTSCSI_REQUEST &&
1619             sc->vtscsi_frozen & VTSCSI_FROZEN_NO_REQUESTS)
1620                 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_NO_REQUESTS;
1621
1622         if (reason & VTSCSI_REQUEST_VQ &&
1623             sc->vtscsi_frozen & VTSCSI_FROZEN_REQUEST_VQ_FULL)
1624                 sc->vtscsi_frozen &= ~VTSCSI_FROZEN_REQUEST_VQ_FULL;
1625
1626         thawed = sc->vtscsi_frozen == 0;
1627         if (thawed != 0)
1628                 vtscsi_dprintf(sc, VTSCSI_INFO, "SIMQ thawed\n");
1629
1630         return (thawed);
1631 }
1632
1633 static void
1634 vtscsi_announce(struct vtscsi_softc *sc, uint32_t ac_code,
1635     target_id_t target_id, lun_id_t lun_id)
1636 {
1637         struct cam_path *path;
1638
1639         /* Use the wildcard path from our softc for bus announcements. */
1640         if (target_id == CAM_TARGET_WILDCARD && lun_id == CAM_LUN_WILDCARD) {
1641                 xpt_async(ac_code, sc->vtscsi_path, NULL);
1642                 return;
1643         }
1644
1645         if (xpt_create_path(&path, NULL, cam_sim_path(sc->vtscsi_sim),
1646             target_id, lun_id) != CAM_REQ_CMP) {
1647                 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot create path\n");
1648                 return;
1649         }
1650
1651         xpt_async(ac_code, path, NULL);
1652         xpt_free_path(path);
1653 }
1654
1655 static void
1656 vtscsi_execute_rescan(struct vtscsi_softc *sc, target_id_t target_id,
1657     lun_id_t lun_id)
1658 {
1659         union ccb *ccb;
1660         cam_status status;
1661
1662         ccb = xpt_alloc_ccb_nowait();
1663         if (ccb == NULL) {
1664                 vtscsi_dprintf(sc, VTSCSI_ERROR, "cannot allocate CCB\n");
1665                 return;
1666         }
1667
1668         status = xpt_create_path(&ccb->ccb_h.path, NULL,
1669             cam_sim_path(sc->vtscsi_sim), target_id, lun_id);
1670         if (status != CAM_REQ_CMP) {
1671                 xpt_free_ccb(ccb);
1672                 return;
1673         }
1674
1675         xpt_rescan(ccb);
1676 }
1677
1678 static void
1679 vtscsi_execute_rescan_bus(struct vtscsi_softc *sc)
1680 {
1681
1682         vtscsi_execute_rescan(sc, CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
1683 }
1684
1685 static void
1686 vtscsi_transport_reset_event(struct vtscsi_softc *sc,
1687     struct virtio_scsi_event *event)
1688 {
1689         target_id_t target_id;
1690         lun_id_t lun_id;
1691
1692         vtscsi_get_request_lun(event->lun, &target_id, &lun_id);
1693
1694         switch (event->reason) {
1695         case VIRTIO_SCSI_EVT_RESET_RESCAN:
1696         case VIRTIO_SCSI_EVT_RESET_REMOVED:
1697                 vtscsi_execute_rescan(sc, target_id, lun_id);
1698                 break;
1699         default:
1700                 device_printf(sc->vtscsi_dev,
1701                     "unhandled transport event reason: %d\n", event->reason);
1702                 break;
1703         }
1704 }
1705
1706 static void
1707 vtscsi_handle_event(struct vtscsi_softc *sc, struct virtio_scsi_event *event)
1708 {
1709         int error;
1710
1711         if ((event->event & VIRTIO_SCSI_T_EVENTS_MISSED) == 0) {
1712                 switch (event->event) {
1713                 case VIRTIO_SCSI_T_TRANSPORT_RESET:
1714                         vtscsi_transport_reset_event(sc, event);
1715                         break;
1716                 default:
1717                         device_printf(sc->vtscsi_dev,
1718                             "unhandled event: %d\n", event->event);
1719                         break;
1720                 }
1721         } else
1722                 vtscsi_execute_rescan_bus(sc);
1723
1724         /*
1725          * This should always be successful since the buffer
1726          * was just dequeued.
1727          */
1728         error = vtscsi_enqueue_event_buf(sc, event);
1729         KASSERT(error == 0,
1730             ("cannot requeue event buffer: %d", error));
1731 }
1732
1733 static int
1734 vtscsi_enqueue_event_buf(struct vtscsi_softc *sc,
1735     struct virtio_scsi_event *event)
1736 {
1737         struct sglist *sg;
1738         struct virtqueue *vq;
1739         int size, error;
1740
1741         sg = sc->vtscsi_sglist;
1742         vq = sc->vtscsi_event_vq;
1743         size = sc->vtscsi_event_buf_size;
1744
1745         bzero(event, size);
1746
1747         sglist_reset(sg);
1748         error = sglist_append(sg, event, size);
1749         if (error)
1750                 return (error);
1751
1752         error = virtqueue_enqueue(vq, event, sg, 0, sg->sg_nseg);
1753         if (error)
1754                 return (error);
1755
1756         virtqueue_notify(vq);
1757
1758         return (0);
1759 }
1760
1761 static int
1762 vtscsi_init_event_vq(struct vtscsi_softc *sc)
1763 {
1764         struct virtio_scsi_event *event;
1765         int i, size, error;
1766
1767         /*
1768          * The first release of QEMU with VirtIO SCSI support would crash
1769          * when attempting to notify the event virtqueue. This was fixed
1770          * when hotplug support was added.
1771          */
1772         if (sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG)
1773                 size = sc->vtscsi_event_buf_size;
1774         else
1775                 size = 0;
1776
1777         if (size < sizeof(struct virtio_scsi_event))
1778                 return (0);
1779
1780         for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) {
1781                 event = &sc->vtscsi_event_bufs[i];
1782
1783                 error = vtscsi_enqueue_event_buf(sc, event);
1784                 if (error)
1785                         break;
1786         }
1787
1788         /*
1789          * Even just one buffer is enough. Missed events are
1790          * denoted with the VIRTIO_SCSI_T_EVENTS_MISSED flag.
1791          */
1792         if (i > 0)
1793                 error = 0;
1794
1795         return (error);
1796 }
1797
1798 static void
1799 vtscsi_reinit_event_vq(struct vtscsi_softc *sc)
1800 {
1801         struct virtio_scsi_event *event;
1802         int i, error;
1803
1804         if ((sc->vtscsi_flags & VTSCSI_FLAG_HOTPLUG) == 0 ||
1805             sc->vtscsi_event_buf_size < sizeof(struct virtio_scsi_event))
1806                 return;
1807
1808         for (i = 0; i < VTSCSI_NUM_EVENT_BUFS; i++) {
1809                 event = &sc->vtscsi_event_bufs[i];
1810
1811                 error = vtscsi_enqueue_event_buf(sc, event);
1812                 if (error)
1813                         break;
1814         }
1815
1816         KASSERT(i > 0, ("cannot reinit event vq: %d", error));
1817 }
1818
1819 static void
1820 vtscsi_drain_event_vq(struct vtscsi_softc *sc)
1821 {
1822         struct virtqueue *vq;
1823         int last;
1824
1825         vq = sc->vtscsi_event_vq;
1826         last = 0;
1827
1828         while (virtqueue_drain(vq, &last) != NULL)
1829                 ;
1830
1831         KASSERT(virtqueue_empty(vq), ("eventvq not empty"));
1832 }
1833
1834 static void
1835 vtscsi_complete_vqs_locked(struct vtscsi_softc *sc)
1836 {
1837
1838         VTSCSI_LOCK_OWNED(sc);
1839
1840         if (sc->vtscsi_request_vq != NULL)
1841                 vtscsi_complete_vq(sc, sc->vtscsi_request_vq);
1842         if (sc->vtscsi_control_vq != NULL)
1843                 vtscsi_complete_vq(sc, sc->vtscsi_control_vq);
1844 }
1845
1846 static void
1847 vtscsi_complete_vqs(struct vtscsi_softc *sc)
1848 {
1849
1850         VTSCSI_LOCK(sc);
1851         vtscsi_complete_vqs_locked(sc);
1852         VTSCSI_UNLOCK(sc);
1853 }
1854
1855 static void
1856 vtscsi_cancel_request(struct vtscsi_softc *sc, struct vtscsi_request *req)
1857 {
1858         union ccb *ccb;
1859         int detach;
1860
1861         ccb = req->vsr_ccb;
1862
1863         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p ccb=%p\n", req, ccb);
1864
1865         /*
1866          * The callout must be drained when detaching since the request is
1867          * about to be freed. The VTSCSI_MTX must not be held for this in
1868          * case the callout is pending because there is a deadlock potential.
1869          * Otherwise, the virtqueue is being drained because of a bus reset
1870          * so we only need to attempt to stop the callouts.
1871          */
1872         detach = (sc->vtscsi_flags & VTSCSI_FLAG_DETACH) != 0;
1873         if (detach != 0)
1874                 VTSCSI_LOCK_NOTOWNED(sc);
1875         else
1876                 VTSCSI_LOCK_OWNED(sc);
1877
1878         if (req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET) {
1879                 if (detach != 0)
1880                         callout_drain(&req->vsr_callout);
1881                 else
1882                         callout_stop(&req->vsr_callout);
1883         }
1884
1885         if (ccb != NULL) {
1886                 if (detach != 0) {
1887                         VTSCSI_LOCK(sc);
1888                         ccb->ccb_h.status = CAM_NO_HBA;
1889                 } else
1890                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
1891                 xpt_done(ccb);
1892                 if (detach != 0)
1893                         VTSCSI_UNLOCK(sc);
1894         }
1895
1896         vtscsi_enqueue_request(sc, req);
1897 }
1898
1899 static void
1900 vtscsi_drain_vq(struct vtscsi_softc *sc, struct virtqueue *vq)
1901 {
1902         struct vtscsi_request *req;
1903         int last;
1904
1905         last = 0;
1906
1907         vtscsi_dprintf(sc, VTSCSI_TRACE, "vq=%p\n", vq);
1908
1909         while ((req = virtqueue_drain(vq, &last)) != NULL)
1910                 vtscsi_cancel_request(sc, req);
1911
1912         KASSERT(virtqueue_empty(vq), ("virtqueue not empty"));
1913 }
1914
1915 static void
1916 vtscsi_drain_vqs(struct vtscsi_softc *sc)
1917 {
1918
1919         if (sc->vtscsi_control_vq != NULL)
1920                 vtscsi_drain_vq(sc, sc->vtscsi_control_vq);
1921         if (sc->vtscsi_request_vq != NULL)
1922                 vtscsi_drain_vq(sc, sc->vtscsi_request_vq);
1923         if (sc->vtscsi_event_vq != NULL)
1924                 vtscsi_drain_event_vq(sc);
1925 }
1926
1927 static void
1928 vtscsi_stop(struct vtscsi_softc *sc)
1929 {
1930
1931         vtscsi_disable_vqs_intr(sc);
1932         virtio_stop(sc->vtscsi_dev);
1933 }
1934
1935 static int
1936 vtscsi_reset_bus(struct vtscsi_softc *sc)
1937 {
1938         int error;
1939
1940         VTSCSI_LOCK_OWNED(sc);
1941
1942         if (vtscsi_bus_reset_disable != 0) {
1943                 device_printf(sc->vtscsi_dev, "bus reset disabled\n");
1944                 return (0);
1945         }
1946
1947         sc->vtscsi_flags |= VTSCSI_FLAG_RESET;
1948
1949         /*
1950          * vtscsi_stop() will cause the in-flight requests to be canceled.
1951          * Those requests are then completed here so CAM will retry them
1952          * after the reset is complete.
1953          */
1954         vtscsi_stop(sc);
1955         vtscsi_complete_vqs_locked(sc);
1956
1957         /* Rid the virtqueues of any remaining requests. */
1958         vtscsi_drain_vqs(sc);
1959
1960         /*
1961          * Any resource shortage that froze the SIMQ cannot persist across
1962          * a bus reset so ensure it gets thawed here.
1963          */
1964         if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST | VTSCSI_REQUEST_VQ) != 0)
1965                 xpt_release_simq(sc->vtscsi_sim, 0);
1966
1967         error = vtscsi_reinit(sc);
1968         if (error) {
1969                 device_printf(sc->vtscsi_dev,
1970                     "reinitialization failed, stopping device...\n");
1971                 vtscsi_stop(sc);
1972         } else
1973                 vtscsi_announce(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
1974                     CAM_LUN_WILDCARD);
1975
1976         sc->vtscsi_flags &= ~VTSCSI_FLAG_RESET;
1977
1978         return (error);
1979 }
1980
1981 static void
1982 vtscsi_init_request(struct vtscsi_softc *sc, struct vtscsi_request *req)
1983 {
1984
1985 #ifdef INVARIANTS
1986         int req_nsegs, resp_nsegs;
1987
1988         req_nsegs = sglist_count(&req->vsr_ureq, sizeof(req->vsr_ureq));
1989         resp_nsegs = sglist_count(&req->vsr_uresp, sizeof(req->vsr_uresp));
1990
1991         KASSERT(req_nsegs == 1, ("request crossed page boundary"));
1992         KASSERT(resp_nsegs == 1, ("response crossed page boundary"));
1993 #endif
1994
1995         req->vsr_softc = sc;
1996         callout_init_mtx(&req->vsr_callout, VTSCSI_MTX(sc), 0);
1997 }
1998
1999 static int
2000 vtscsi_alloc_requests(struct vtscsi_softc *sc)
2001 {
2002         struct vtscsi_request *req;
2003         int i, nreqs;
2004
2005         /*
2006          * Commands destined for either the request or control queues come
2007          * from the same SIM queue. Use the size of the request virtqueue
2008          * as it (should) be much more frequently used. Some additional
2009          * requests are allocated for internal (TMF) use.
2010          */
2011         nreqs = virtqueue_size(sc->vtscsi_request_vq);
2012         if ((sc->vtscsi_flags & VTSCSI_FLAG_INDIRECT) == 0)
2013                 nreqs /= VTSCSI_MIN_SEGMENTS;
2014         nreqs += VTSCSI_RESERVED_REQUESTS;
2015
2016         for (i = 0; i < nreqs; i++) {
2017                 req = malloc(sizeof(struct vtscsi_request), M_DEVBUF,
2018                     M_NOWAIT);
2019                 if (req == NULL)
2020                         return (ENOMEM);
2021
2022                 vtscsi_init_request(sc, req);
2023
2024                 sc->vtscsi_nrequests++;
2025                 vtscsi_enqueue_request(sc, req);
2026         }
2027
2028         return (0);
2029 }
2030
2031 static void
2032 vtscsi_free_requests(struct vtscsi_softc *sc)
2033 {
2034         struct vtscsi_request *req;
2035
2036         while ((req = vtscsi_dequeue_request(sc)) != NULL) {
2037                 KASSERT(callout_active(&req->vsr_callout) == 0,
2038                     ("request callout still active"));
2039
2040                 sc->vtscsi_nrequests--;
2041                 free(req, M_DEVBUF);
2042         }
2043
2044         KASSERT(sc->vtscsi_nrequests == 0, ("leaked requests: %d",
2045             sc->vtscsi_nrequests));
2046 }
2047
2048 static void
2049 vtscsi_enqueue_request(struct vtscsi_softc *sc, struct vtscsi_request *req)
2050 {
2051
2052         KASSERT(req->vsr_softc == sc,
2053             ("non-matching request vsr_softc %p/%p", req->vsr_softc, sc));
2054
2055         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req);
2056
2057         /* A request is available so the SIMQ could be released. */
2058         if (vtscsi_thaw_simq(sc, VTSCSI_REQUEST) != 0)
2059                 xpt_release_simq(sc->vtscsi_sim, 1);
2060
2061         req->vsr_ccb = NULL;
2062         req->vsr_complete = NULL;
2063         req->vsr_ptr0 = NULL;
2064         req->vsr_state = VTSCSI_REQ_STATE_FREE;
2065         req->vsr_flags = 0;
2066
2067         bzero(&req->vsr_ureq, sizeof(req->vsr_ureq));
2068         bzero(&req->vsr_uresp, sizeof(req->vsr_uresp));
2069
2070         /*
2071          * We insert at the tail of the queue in order to make it
2072          * very unlikely a request will be reused if we race with
2073          * stopping its callout handler.
2074          */
2075         TAILQ_INSERT_TAIL(&sc->vtscsi_req_free, req, vsr_link);
2076 }
2077
2078 static struct vtscsi_request *
2079 vtscsi_dequeue_request(struct vtscsi_softc *sc)
2080 {
2081         struct vtscsi_request *req;
2082
2083         req = TAILQ_FIRST(&sc->vtscsi_req_free);
2084         if (req != NULL) {
2085                 req->vsr_state = VTSCSI_REQ_STATE_INUSE;
2086                 TAILQ_REMOVE(&sc->vtscsi_req_free, req, vsr_link);
2087         } else
2088                 sc->vtscsi_stats.dequeue_no_requests++;
2089
2090         vtscsi_dprintf(sc, VTSCSI_TRACE, "req=%p\n", req);
2091
2092         return (req);
2093 }
2094
2095 static void
2096 vtscsi_complete_request(struct vtscsi_request *req)
2097 {
2098
2099         if (req->vsr_flags & VTSCSI_REQ_FLAG_POLLED)
2100                 req->vsr_flags |= VTSCSI_REQ_FLAG_COMPLETE;
2101
2102         if (req->vsr_complete != NULL)
2103                 req->vsr_complete(req->vsr_softc, req);
2104 }
2105
2106 static void
2107 vtscsi_complete_vq(struct vtscsi_softc *sc, struct virtqueue *vq)
2108 {
2109         struct vtscsi_request *req;
2110
2111         VTSCSI_LOCK_OWNED(sc);
2112
2113         while ((req = virtqueue_dequeue(vq, NULL)) != NULL)
2114                 vtscsi_complete_request(req);
2115 }
2116
2117 static void
2118 vtscsi_control_vq_intr(void *xsc)
2119 {
2120         struct vtscsi_softc *sc;
2121         struct virtqueue *vq;
2122
2123         sc = xsc;
2124         vq = sc->vtscsi_control_vq;
2125
2126 again:
2127         VTSCSI_LOCK(sc);
2128
2129         vtscsi_complete_vq(sc, sc->vtscsi_control_vq);
2130
2131         if (virtqueue_enable_intr(vq) != 0) {
2132                 virtqueue_disable_intr(vq);
2133                 VTSCSI_UNLOCK(sc);
2134                 goto again;
2135         }
2136
2137         VTSCSI_UNLOCK(sc);
2138 }
2139
2140 static void
2141 vtscsi_event_vq_intr(void *xsc)
2142 {
2143         struct vtscsi_softc *sc;
2144         struct virtqueue *vq;
2145         struct virtio_scsi_event *event;
2146
2147         sc = xsc;
2148         vq = sc->vtscsi_event_vq;
2149
2150 again:
2151         VTSCSI_LOCK(sc);
2152
2153         while ((event = virtqueue_dequeue(vq, NULL)) != NULL)
2154                 vtscsi_handle_event(sc, event);
2155
2156         if (virtqueue_enable_intr(vq) != 0) {
2157                 virtqueue_disable_intr(vq);
2158                 VTSCSI_UNLOCK(sc);
2159                 goto again;
2160         }
2161
2162         VTSCSI_UNLOCK(sc);
2163 }
2164
2165 static void
2166 vtscsi_request_vq_intr(void *xsc)
2167 {
2168         struct vtscsi_softc *sc;
2169         struct virtqueue *vq;
2170
2171         sc = xsc;
2172         vq = sc->vtscsi_request_vq;
2173
2174 again:
2175         VTSCSI_LOCK(sc);
2176
2177         vtscsi_complete_vq(sc, sc->vtscsi_request_vq);
2178
2179         if (virtqueue_enable_intr(vq) != 0) {
2180                 virtqueue_disable_intr(vq);
2181                 VTSCSI_UNLOCK(sc);
2182                 goto again;
2183         }
2184
2185         VTSCSI_UNLOCK(sc);
2186 }
2187
2188 static void
2189 vtscsi_disable_vqs_intr(struct vtscsi_softc *sc)
2190 {
2191
2192         virtqueue_disable_intr(sc->vtscsi_control_vq);
2193         virtqueue_disable_intr(sc->vtscsi_event_vq);
2194         virtqueue_disable_intr(sc->vtscsi_request_vq);
2195 }
2196
2197 static void
2198 vtscsi_enable_vqs_intr(struct vtscsi_softc *sc)
2199 {
2200
2201         virtqueue_enable_intr(sc->vtscsi_control_vq);
2202         virtqueue_enable_intr(sc->vtscsi_event_vq);
2203         virtqueue_enable_intr(sc->vtscsi_request_vq);
2204 }
2205
2206 static void
2207 vtscsi_get_tunables(struct vtscsi_softc *sc)
2208 {
2209         char tmpstr[64];
2210
2211         TUNABLE_INT_FETCH("hw.vtscsi.debug_level", &sc->vtscsi_debug);
2212
2213         snprintf(tmpstr, sizeof(tmpstr), "dev.vtscsi.%d.debug_level",
2214             device_get_unit(sc->vtscsi_dev));
2215         TUNABLE_INT_FETCH(tmpstr, &sc->vtscsi_debug);
2216 }
2217
2218 static void
2219 vtscsi_add_sysctl(struct vtscsi_softc *sc)
2220 {
2221         device_t dev;
2222         struct vtscsi_statistics *stats;
2223         struct sysctl_ctx_list *ctx;
2224         struct sysctl_oid *tree;
2225         struct sysctl_oid_list *child;
2226
2227         dev = sc->vtscsi_dev;
2228         stats = &sc->vtscsi_stats;
2229         ctx = device_get_sysctl_ctx(dev);
2230         tree = device_get_sysctl_tree(dev);
2231         child = SYSCTL_CHILDREN(tree);
2232
2233         SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level",
2234             CTLFLAG_RW, &sc->vtscsi_debug, 0,
2235             "Debug level");
2236
2237         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "scsi_cmd_timeouts",
2238             CTLFLAG_RD, &stats->scsi_cmd_timeouts,
2239             "SCSI command timeouts");
2240         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "dequeue_no_requests",
2241             CTLFLAG_RD, &stats->dequeue_no_requests,
2242             "No available requests to dequeue");
2243 }
2244
2245 static void
2246 vtscsi_printf_req(struct vtscsi_request *req, const char *func,
2247     const char *fmt, ...)
2248 {
2249         struct vtscsi_softc *sc;
2250         union ccb *ccb;
2251         struct sbuf sb;
2252         va_list ap;
2253         char str[192];
2254         char path_str[64];
2255
2256         if (req == NULL)
2257                 return;
2258
2259         sc = req->vsr_softc;
2260         ccb = req->vsr_ccb;
2261
2262         va_start(ap, fmt);
2263         sbuf_new(&sb, str, sizeof(str), 0);
2264
2265         if (ccb == NULL) {
2266                 sbuf_printf(&sb, "(noperiph:%s%d:%u): ",
2267                     cam_sim_name(sc->vtscsi_sim), cam_sim_unit(sc->vtscsi_sim),
2268                     cam_sim_bus(sc->vtscsi_sim));
2269         } else {
2270                 xpt_path_string(ccb->ccb_h.path, path_str, sizeof(path_str));
2271                 sbuf_cat(&sb, path_str);
2272                 if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
2273                         scsi_command_string(&ccb->csio, &sb);
2274                         sbuf_printf(&sb, "length %d ", ccb->csio.dxfer_len);
2275                 }
2276         }
2277
2278         sbuf_vprintf(&sb, fmt, ap);
2279         va_end(ap);
2280
2281         sbuf_finish(&sb);
2282         printf("%s: %s: %s", device_get_nameunit(sc->vtscsi_dev), func,
2283             sbuf_data(&sb));
2284 }