sys/dev/hyperv/storvsc/hv_storvsc_drv_freebsd.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2009-2012,2016-2017 Microsoft Corp.
   5  * Copyright (c) 2012 NetApp Inc.
   6  * Copyright (c) 2012 Citrix Inc.
   7  * All rights reserved.
   8  *
   9  * Redistribution and use in source and binary forms, with or without
  10  * modification, are permitted provided that the following conditions
  11  * are met:
  12  * 1. Redistributions of source code must retain the above copyright
  13  *    notice unmodified, this list of conditions, and the following
  14  *    disclaimer.
  15  * 2. Redistributions in binary form must reproduce the above copyright
  16  *    notice, this list of conditions and the following disclaimer in the
  17  *    documentation and/or other materials provided with the distribution.
  18  *
  19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  20  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  21  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  22  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  23  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  24  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  28  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29  */
  30
  31 /**
  32  * StorVSC driver for Hyper-V.  This driver presents a SCSI HBA interface
  33  * to the Comman Access Method (CAM) layer.  CAM control blocks (CCBs) are
  34  * converted into VSCSI protocol messages which are delivered to the parent
  35  * partition StorVSP driver over the Hyper-V VMBUS.
  36  */
  37 #include <sys/cdefs.h>
  38 __FBSDID("$FreeBSD$");
  39
  40 #include <sys/param.h>
  41 #include <sys/proc.h>
  42 #include <sys/condvar.h>
  43 #include <sys/time.h>
  44 #include <sys/systm.h>
  45 #include <sys/sysctl.h>
  46 #include <sys/sockio.h>
  47 #include <sys/mbuf.h>
  48 #include <sys/malloc.h>
  49 #include <sys/module.h>
  50 #include <sys/kernel.h>
  51 #include <sys/queue.h>
  52 #include <sys/lock.h>
  53 #include <sys/sx.h>
  54 #include <sys/taskqueue.h>
  55 #include <sys/bus.h>
  56 #include <sys/mutex.h>
  57 #include <sys/callout.h>
  58 #include <sys/smp.h>
  59 #include <vm/vm.h>
  60 #include <vm/pmap.h>
  61 #include <vm/uma.h>
  62 #include <sys/lock.h>
  63 #include <sys/sema.h>
  64 #include <sys/sglist.h>
  65 #include <sys/eventhandler.h>
  66 #include <machine/bus.h>
  67
  68 #include <cam/cam.h>
  69 #include <cam/cam_ccb.h>
  70 #include <cam/cam_periph.h>
  71 #include <cam/cam_sim.h>
  72 #include <cam/cam_xpt_sim.h>
  73 #include <cam/cam_xpt_internal.h>
  74 #include <cam/cam_debug.h>
  75 #include <cam/scsi/scsi_all.h>
  76 #include <cam/scsi/scsi_message.h>
  77
  78 #include <dev/hyperv/include/hyperv.h>
  79 #include <dev/hyperv/include/vmbus.h>
  80 #include "hv_vstorage.h"
  81 #include "vmbus_if.h"
  82
  83 #define STORVSC_MAX_LUNS_PER_TARGET     (64)
  84 #define STORVSC_MAX_IO_REQUESTS         (STORVSC_MAX_LUNS_PER_TARGET * 2)
  85 #define BLKVSC_MAX_IDE_DISKS_PER_TARGET (1)
  86 #define BLKVSC_MAX_IO_REQUESTS          STORVSC_MAX_IO_REQUESTS
  87 #define STORVSC_MAX_TARGETS             (2)
  88
  89 #define VSTOR_PKT_SIZE  (sizeof(struct vstor_packet) - vmscsi_size_delta)
  90
  91 /*
  92  * 33 segments are needed to allow 128KB maxio, in case the data
  93  * in the first page is _not_ PAGE_SIZE aligned, e.g.
  94  *
  95  *     |<----------- 128KB ----------->|
  96  *     |                               |
  97  *  0  2K 4K    8K   16K   124K  128K  130K
  98  *  |  |  |     |     |       |     |  |
  99  *  +--+--+-----+-----+.......+-----+--+--+
 100  *  |  |  |     |     |       |     |  |  | DATA
 101  *  |  |  |     |     |       |     |  |  |
 102  *  +--+--+-----+-----+.......------+--+--+
 103  *     |  |                         |  |
 104  *     | 1|            31           | 1| ...... # of segments
 105  */
 106 #define STORVSC_DATA_SEGCNT_MAX         33
 107 #define STORVSC_DATA_SEGSZ_MAX          PAGE_SIZE
 108 #define STORVSC_DATA_SIZE_MAX           \
 109         ((STORVSC_DATA_SEGCNT_MAX - 1) * STORVSC_DATA_SEGSZ_MAX)
 110
 111 struct storvsc_softc;
 112
 113 struct hv_sgl_node {
 114         LIST_ENTRY(hv_sgl_node) link;
 115         struct sglist *sgl_data;
 116 };
 117
 118 struct hv_sgl_page_pool{
 119         LIST_HEAD(, hv_sgl_node) in_use_sgl_list;
 120         LIST_HEAD(, hv_sgl_node) free_sgl_list;
 121         boolean_t                is_init;
 122 } g_hv_sgl_page_pool;
 123
 124 enum storvsc_request_type {
 125         WRITE_TYPE,
 126         READ_TYPE,
 127         UNKNOWN_TYPE
 128 };
 129
 130 SYSCTL_NODE(_hw, OID_AUTO, storvsc, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL,
 131         "Hyper-V storage interface");
 132
 133 static u_int hv_storvsc_use_win8ext_flags = 1;
 134 SYSCTL_UINT(_hw_storvsc, OID_AUTO, use_win8ext_flags, CTLFLAG_RW,
 135         &hv_storvsc_use_win8ext_flags, 0,
 136         "Use win8 extension flags or not");
 137
 138 static u_int hv_storvsc_use_pim_unmapped = 1;
 139 SYSCTL_UINT(_hw_storvsc, OID_AUTO, use_pim_unmapped, CTLFLAG_RDTUN,
 140         &hv_storvsc_use_pim_unmapped, 0,
 141         "Optimize storvsc by using unmapped I/O");
 142
 143 static u_int hv_storvsc_ringbuffer_size = (64 * PAGE_SIZE);
 144 SYSCTL_UINT(_hw_storvsc, OID_AUTO, ringbuffer_size, CTLFLAG_RDTUN,
 145         &hv_storvsc_ringbuffer_size, 0, "Hyper-V storage ringbuffer size");
 146
 147 static u_int hv_storvsc_max_io = 512;
 148 SYSCTL_UINT(_hw_storvsc, OID_AUTO, max_io, CTLFLAG_RDTUN,
 149         &hv_storvsc_max_io, 0, "Hyper-V storage max io limit");
 150
 151 static int hv_storvsc_chan_cnt = 0;
 152 SYSCTL_INT(_hw_storvsc, OID_AUTO, chan_cnt, CTLFLAG_RDTUN,
 153         &hv_storvsc_chan_cnt, 0, "# of channels to use");
 154
 155 #define STORVSC_MAX_IO                                          \
 156         vmbus_chan_prplist_nelem(hv_storvsc_ringbuffer_size,    \
 157            STORVSC_DATA_SEGCNT_MAX, VSTOR_PKT_SIZE)
 158
 159 struct hv_storvsc_sysctl {
 160         u_long          data_bio_cnt;
 161         u_long          data_vaddr_cnt;
 162         u_long          data_sg_cnt;
 163         u_long          chan_send_cnt[MAXCPU];
 164 };
 165
 166 struct storvsc_gpa_range {
 167         struct vmbus_gpa_range  gpa_range;
 168         uint64_t                gpa_page[STORVSC_DATA_SEGCNT_MAX];
 169 } __packed;
 170
 171 struct hv_storvsc_request {
 172         LIST_ENTRY(hv_storvsc_request)  link;
 173         struct vstor_packet             vstor_packet;
 174         int                             prp_cnt;
 175         struct storvsc_gpa_range        prp_list;
 176         void                            *sense_data;
 177         uint8_t                         sense_info_len;
 178         uint8_t                         retries;
 179         union ccb                       *ccb;
 180         struct storvsc_softc            *softc;
 181         struct callout                  callout;
 182         struct sema                     synch_sema; /*Synchronize the request/response if needed */
 183         struct sglist                   *bounce_sgl;
 184         unsigned int                    bounce_sgl_count;
 185         uint64_t                        not_aligned_seg_bits;
 186         bus_dmamap_t                    data_dmap;
 187 };
 188
 189 struct storvsc_softc {
 190         struct vmbus_channel            *hs_chan;
 191         LIST_HEAD(, hv_storvsc_request) hs_free_list;
 192         struct mtx                      hs_lock;
 193         struct storvsc_driver_props     *hs_drv_props;
 194         int                             hs_unit;
 195         uint32_t                        hs_frozen;
 196         struct cam_sim                  *hs_sim;
 197         struct cam_path                 *hs_path;
 198         uint32_t                        hs_num_out_reqs;
 199         boolean_t                       hs_destroy;
 200         boolean_t                       hs_drain_notify;
 201         struct sema                     hs_drain_sema;
 202         struct hv_storvsc_request       hs_init_req;
 203         struct hv_storvsc_request       hs_reset_req;
 204         device_t                        hs_dev;
 205         bus_dma_tag_t                   storvsc_req_dtag;
 206         struct hv_storvsc_sysctl        sysctl_data;
 207         uint32_t                        hs_nchan;
 208         struct vmbus_channel            *hs_sel_chan[MAXCPU];
 209 };
 210
 211 static eventhandler_tag storvsc_handler_tag;
 212 /*
 213  * The size of the vmscsi_request has changed in win8. The
 214  * additional size is for the newly added elements in the
 215  * structure. These elements are valid only when we are talking
 216  * to a win8 host.
 217  * Track the correct size we need to apply.
 218  */
 219 static int vmscsi_size_delta = sizeof(struct vmscsi_win8_extension);
 220
 221 /**
 222  * HyperV storvsc timeout testing cases:
 223  * a. IO returned after first timeout;
 224  * b. IO returned after second timeout and queue freeze;
 225  * c. IO returned while timer handler is running
 226  * The first can be tested by "sg_senddiag -vv /dev/daX",
 227  * and the second and third can be done by
 228  * "sg_wr_mode -v -p 08 -c 0,1a -m 0,ff /dev/daX".
 229  */
 230 #define HVS_TIMEOUT_TEST 0
 231
 232 /*
 233  * Bus/adapter reset functionality on the Hyper-V host is
 234  * buggy and it will be disabled until
 235  * it can be further tested.
 236  */
 237 #define HVS_HOST_RESET 0
 238
 239 struct storvsc_driver_props {
 240         char            *drv_name;
 241         char            *drv_desc;
 242         uint8_t         drv_max_luns_per_target;
 243         uint32_t        drv_max_ios_per_target;
 244         uint32_t        drv_ringbuffer_size;
 245 };
 246
 247 enum hv_storage_type {
 248         DRIVER_BLKVSC,
 249         DRIVER_STORVSC,
 250         DRIVER_UNKNOWN
 251 };
 252
 253 #define HS_MAX_ADAPTERS 10
 254
 255 #define HV_STORAGE_SUPPORTS_MULTI_CHANNEL 0x1
 256
 257 /* {ba6163d9-04a1-4d29-b605-72e2ffb1dc7f} */
 258 static const struct hyperv_guid gStorVscDeviceType={
 259         .hv_guid = {0xd9, 0x63, 0x61, 0xba, 0xa1, 0x04, 0x29, 0x4d,
 260                  0xb6, 0x05, 0x72, 0xe2, 0xff, 0xb1, 0xdc, 0x7f}
 261 };
 262
 263 /* {32412632-86cb-44a2-9b5c-50d1417354f5} */
 264 static const struct hyperv_guid gBlkVscDeviceType={
 265         .hv_guid = {0x32, 0x26, 0x41, 0x32, 0xcb, 0x86, 0xa2, 0x44,
 266                  0x9b, 0x5c, 0x50, 0xd1, 0x41, 0x73, 0x54, 0xf5}
 267 };
 268
 269 static struct storvsc_driver_props g_drv_props_table[] = {
 270         {"blkvsc", "Hyper-V IDE",
 271          BLKVSC_MAX_IDE_DISKS_PER_TARGET, BLKVSC_MAX_IO_REQUESTS,
 272          20*PAGE_SIZE},
 273         {"storvsc", "Hyper-V SCSI",
 274          STORVSC_MAX_LUNS_PER_TARGET, STORVSC_MAX_IO_REQUESTS,
 275          20*PAGE_SIZE}
 276 };
 277
 278 /*
 279  * Sense buffer size changed in win8; have a run-time
 280  * variable to track the size we should use.
 281  */
 282 static int sense_buffer_size = PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE;
 283
 284 /*
 285  * The storage protocol version is determined during the
 286  * initial exchange with the host.  It will indicate which
 287  * storage functionality is available in the host.
 288 */
 289 static int vmstor_proto_version;
 290
 291 struct vmstor_proto {
 292         int proto_version;
 293         int sense_buffer_size;
 294         int vmscsi_size_delta;
 295 };
 296
 297 static const struct vmstor_proto vmstor_proto_list[] = {
 298         {
 299                 VMSTOR_PROTOCOL_VERSION_WIN10,
 300                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 301                 0
 302         },
 303         {
 304                 VMSTOR_PROTOCOL_VERSION_WIN8_1,
 305                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 306                 0
 307         },
 308         {
 309                 VMSTOR_PROTOCOL_VERSION_WIN8,
 310                 POST_WIN7_STORVSC_SENSE_BUFFER_SIZE,
 311                 0
 312         },
 313         {
 314                 VMSTOR_PROTOCOL_VERSION_WIN7,
 315                 PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE,
 316                 sizeof(struct vmscsi_win8_extension),
 317         },
 318         {
 319                 VMSTOR_PROTOCOL_VERSION_WIN6,
 320                 PRE_WIN8_STORVSC_SENSE_BUFFER_SIZE,
 321                 sizeof(struct vmscsi_win8_extension),
 322         }
 323 };
 324
 325 /* static functions */
 326 static int storvsc_probe(device_t dev);
 327 static int storvsc_attach(device_t dev);
 328 static int storvsc_detach(device_t dev);
 329 static void storvsc_poll(struct cam_sim * sim);
 330 static void storvsc_action(struct cam_sim * sim, union ccb * ccb);
 331 static int create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp);
 332 static void storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp);
 333 static enum hv_storage_type storvsc_get_storage_type(device_t dev);
 334 static void hv_storvsc_rescan_target(struct storvsc_softc *sc);
 335 static void hv_storvsc_on_channel_callback(struct vmbus_channel *chan, void *xsc);
 336 static void hv_storvsc_on_iocompletion( struct storvsc_softc *sc,
 337                                         struct vstor_packet *vstor_packet,
 338                                         struct hv_storvsc_request *request);
 339 static int hv_storvsc_connect_vsp(struct storvsc_softc *);
 340 static void storvsc_io_done(struct hv_storvsc_request *reqp);
 341 static void storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
 342                                 bus_dma_segment_t *orig_sgl,
 343                                 unsigned int orig_sgl_count,
 344                                 uint64_t seg_bits);
 345 void storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
 346                                 unsigned int dest_sgl_count,
 347                                 struct sglist* src_sgl,
 348                                 uint64_t seg_bits);
 349
 350 static device_method_t storvsc_methods[] = {
 351         /* Device interface */
 352         DEVMETHOD(device_probe,         storvsc_probe),
 353         DEVMETHOD(device_attach,        storvsc_attach),
 354         DEVMETHOD(device_detach,        storvsc_detach),
 355         DEVMETHOD(device_shutdown,      bus_generic_shutdown),
 356         DEVMETHOD_END
 357 };
 358
 359 static driver_t storvsc_driver = {
 360         "storvsc", storvsc_methods, sizeof(struct storvsc_softc),
 361 };
 362
 363 static devclass_t storvsc_devclass;
 364 DRIVER_MODULE(storvsc, vmbus, storvsc_driver, storvsc_devclass, 0, 0);
 365 MODULE_VERSION(storvsc, 1);
 366 MODULE_DEPEND(storvsc, vmbus, 1, 1, 1);
 367
 368 static void
 369 storvsc_subchan_attach(struct storvsc_softc *sc,
 370     struct vmbus_channel *new_channel)
 371 {
 372         struct vmstor_chan_props props;
 373         int ret = 0;
 374
 375         memset(&props, 0, sizeof(props));
 376
 377         vmbus_chan_cpu_rr(new_channel);
 378         ret = vmbus_chan_open(new_channel,
 379             sc->hs_drv_props->drv_ringbuffer_size,
 380             sc->hs_drv_props->drv_ringbuffer_size,
 381             (void *)&props,
 382             sizeof(struct vmstor_chan_props),
 383             hv_storvsc_on_channel_callback, sc);
 384 }
 385
 386 /**
 387  * @brief Send multi-channel creation request to host
 388  *
 389  * @param device  a Hyper-V device pointer
 390  * @param max_chans  the max channels supported by vmbus
 391  */
 392 static void
 393 storvsc_send_multichannel_request(struct storvsc_softc *sc, int max_subch)
 394 {
 395         struct vmbus_channel **subchan;
 396         struct hv_storvsc_request *request;
 397         struct vstor_packet *vstor_packet;
 398         int request_subch;
 399         int ret, i;
 400
 401         /* get sub-channel count that need to create */
 402         request_subch = MIN(max_subch, mp_ncpus - 1);
 403
 404         request = &sc->hs_init_req;
 405
 406         /* request the host to create multi-channel */
 407         memset(request, 0, sizeof(struct hv_storvsc_request));
 408
 409         sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
 410
 411         vstor_packet = &request->vstor_packet;
 412
 413         vstor_packet->operation = VSTOR_OPERATION_CREATE_MULTI_CHANNELS;
 414         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 415         vstor_packet->u.multi_channels_cnt = request_subch;
 416
 417         ret = vmbus_chan_send(sc->hs_chan,
 418             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 419             vstor_packet, VSTOR_PKT_SIZE, (uint64_t)(uintptr_t)request);
 420
 421         sema_wait(&request->synch_sema);
 422
 423         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 424             vstor_packet->status != 0) {
 425                 printf("Storvsc_error: create multi-channel invalid operation "
 426                     "(%d) or statue (%u)\n",
 427                     vstor_packet->operation, vstor_packet->status);
 428                 return;
 429         }
 430
 431         /* Update channel count */
 432         sc->hs_nchan = request_subch + 1;
 433
 434         /* Wait for sub-channels setup to complete. */
 435         subchan = vmbus_subchan_get(sc->hs_chan, request_subch);
 436
 437         /* Attach the sub-channels. */
 438         for (i = 0; i < request_subch; ++i)
 439                 storvsc_subchan_attach(sc, subchan[i]);
 440
 441         /* Release the sub-channels. */
 442         vmbus_subchan_rel(subchan, request_subch);
 443
 444         if (bootverbose)
 445                 printf("Storvsc create multi-channel success!\n");
 446 }
 447
 448 /**
 449  * @brief initialize channel connection to parent partition
 450  *
 451  * @param dev  a Hyper-V device pointer
 452  * @returns  0 on success, non-zero error on failure
 453  */
 454 static int
 455 hv_storvsc_channel_init(struct storvsc_softc *sc)
 456 {
 457         int ret = 0, i;
 458         struct hv_storvsc_request *request;
 459         struct vstor_packet *vstor_packet;
 460         uint16_t max_subch;
 461         boolean_t support_multichannel;
 462         uint32_t version;
 463
 464         max_subch = 0;
 465         support_multichannel = FALSE;
 466
 467         request = &sc->hs_init_req;
 468         memset(request, 0, sizeof(struct hv_storvsc_request));
 469         vstor_packet = &request->vstor_packet;
 470         request->softc = sc;
 471
 472         /**
 473          * Initiate the vsc/vsp initialization protocol on the open channel
 474          */
 475         sema_init(&request->synch_sema, 0, ("stor_synch_sema"));
 476
 477         vstor_packet->operation = VSTOR_OPERATION_BEGININITIALIZATION;
 478         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 479
 480
 481         ret = vmbus_chan_send(sc->hs_chan,
 482             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 483             vstor_packet, VSTOR_PKT_SIZE, (uint64_t)(uintptr_t)request);
 484
 485         if (ret != 0)
 486                 goto cleanup;
 487
 488         sema_wait(&request->synch_sema);
 489
 490         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 491                 vstor_packet->status != 0) {
 492                 goto cleanup;
 493         }
 494
 495         for (i = 0; i < nitems(vmstor_proto_list); i++) {
 496                 /* reuse the packet for version range supported */
 497
 498                 memset(vstor_packet, 0, sizeof(struct vstor_packet));
 499                 vstor_packet->operation = VSTOR_OPERATION_QUERYPROTOCOLVERSION;
 500                 vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 501
 502                 vstor_packet->u.version.major_minor =
 503                         vmstor_proto_list[i].proto_version;
 504
 505                 /* revision is only significant for Windows guests */
 506                 vstor_packet->u.version.revision = 0;
 507
 508                 ret = vmbus_chan_send(sc->hs_chan,
 509                     VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 510                     vstor_packet, VSTOR_PKT_SIZE, (uint64_t)(uintptr_t)request);
 511
 512                 if (ret != 0)
 513                         goto cleanup;
 514
 515                 sema_wait(&request->synch_sema);
 516
 517                 if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO) {
 518                         ret = EINVAL;
 519                         goto cleanup;
 520                 }
 521                 if (vstor_packet->status == 0) {
 522                         vmstor_proto_version =
 523                                 vmstor_proto_list[i].proto_version;
 524                         sense_buffer_size =
 525                                 vmstor_proto_list[i].sense_buffer_size;
 526                         vmscsi_size_delta =
 527                                 vmstor_proto_list[i].vmscsi_size_delta;
 528                         break;
 529                 }
 530         }
 531
 532         if (vstor_packet->status != 0) {
 533                 ret = EINVAL;
 534                 goto cleanup;
 535         }
 536         /**
 537          * Query channel properties
 538          */
 539         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 540         vstor_packet->operation = VSTOR_OPERATION_QUERYPROPERTIES;
 541         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 542
 543         ret = vmbus_chan_send(sc->hs_chan,
 544             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 545             vstor_packet, VSTOR_PKT_SIZE, (uint64_t)(uintptr_t)request);
 546
 547         if ( ret != 0)
 548                 goto cleanup;
 549
 550         sema_wait(&request->synch_sema);
 551
 552         /* TODO: Check returned version */
 553         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 554             vstor_packet->status != 0) {
 555                 goto cleanup;
 556         }
 557
 558         max_subch = vstor_packet->u.chan_props.max_channel_cnt;
 559         if (hv_storvsc_chan_cnt > 0 && hv_storvsc_chan_cnt < (max_subch + 1))
 560                 max_subch = hv_storvsc_chan_cnt - 1;
 561
 562         /* multi-channels feature is supported by WIN8 and above version */
 563         version = VMBUS_GET_VERSION(device_get_parent(sc->hs_dev), sc->hs_dev);
 564         if (version != VMBUS_VERSION_WIN7 && version != VMBUS_VERSION_WS2008 &&
 565             (vstor_packet->u.chan_props.flags &
 566              HV_STORAGE_SUPPORTS_MULTI_CHANNEL)) {
 567                 support_multichannel = TRUE;
 568         }
 569         if (bootverbose) {
 570                 device_printf(sc->hs_dev, "max chans %d%s\n", max_subch + 1,
 571                     support_multichannel ? ", multi-chan capable" : "");
 572         }
 573
 574         memset(vstor_packet, 0, sizeof(struct vstor_packet));
 575         vstor_packet->operation = VSTOR_OPERATION_ENDINITIALIZATION;
 576         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 577
 578         ret = vmbus_chan_send(sc->hs_chan,
 579             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 580             vstor_packet, VSTOR_PKT_SIZE, (uint64_t)(uintptr_t)request);
 581
 582         if (ret != 0) {
 583                 goto cleanup;
 584         }
 585
 586         sema_wait(&request->synch_sema);
 587
 588         if (vstor_packet->operation != VSTOR_OPERATION_COMPLETEIO ||
 589             vstor_packet->status != 0)
 590                 goto cleanup;
 591
 592         /*
 593          * If multi-channel is supported, send multichannel create
 594          * request to host.
 595          */
 596         if (support_multichannel && max_subch > 0)
 597                 storvsc_send_multichannel_request(sc, max_subch);
 598 cleanup:
 599         sema_destroy(&request->synch_sema);
 600         return (ret);
 601 }
 602
 603 /**
 604  * @brief Open channel connection to paraent partition StorVSP driver
 605  *
 606  * Open and initialize channel connection to parent partition StorVSP driver.
 607  *
 608  * @param pointer to a Hyper-V device
 609  * @returns 0 on success, non-zero error on failure
 610  */
 611 static int
 612 hv_storvsc_connect_vsp(struct storvsc_softc *sc)
 613 {
 614         int ret = 0;
 615         struct vmstor_chan_props props;
 616
 617         memset(&props, 0, sizeof(struct vmstor_chan_props));
 618
 619         /*
 620          * Open the channel
 621          */
 622         vmbus_chan_cpu_rr(sc->hs_chan);
 623         ret = vmbus_chan_open(
 624                 sc->hs_chan,
 625                 sc->hs_drv_props->drv_ringbuffer_size,
 626                 sc->hs_drv_props->drv_ringbuffer_size,
 627                 (void *)&props,
 628                 sizeof(struct vmstor_chan_props),
 629                 hv_storvsc_on_channel_callback, sc);
 630
 631         if (ret != 0) {
 632                 return ret;
 633         }
 634
 635         ret = hv_storvsc_channel_init(sc);
 636         return (ret);
 637 }
 638
 639 #if HVS_HOST_RESET
 640 static int
 641 hv_storvsc_host_reset(struct storvsc_softc *sc)
 642 {
 643         int ret = 0;
 644
 645         struct hv_storvsc_request *request;
 646         struct vstor_packet *vstor_packet;
 647
 648         request = &sc->hs_reset_req;
 649         request->softc = sc;
 650         vstor_packet = &request->vstor_packet;
 651
 652         sema_init(&request->synch_sema, 0, "stor synch sema");
 653
 654         vstor_packet->operation = VSTOR_OPERATION_RESETBUS;
 655         vstor_packet->flags = REQUEST_COMPLETION_FLAG;
 656
 657         ret = vmbus_chan_send(dev->channel,
 658             VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 659             vstor_packet, VSTOR_PKT_SIZE,
 660             (uint64_t)(uintptr_t)&sc->hs_reset_req);
 661
 662         if (ret != 0) {
 663                 goto cleanup;
 664         }
 665
 666         sema_wait(&request->synch_sema);
 667
 668         /*
 669          * At this point, all outstanding requests in the adapter
 670          * should have been flushed out and return to us
 671          */
 672
 673 cleanup:
 674         sema_destroy(&request->synch_sema);
 675         return (ret);
 676 }
 677 #endif /* HVS_HOST_RESET */
 678
 679 /**
 680  * @brief Function to initiate an I/O request
 681  *
 682  * @param device Hyper-V device pointer
 683  * @param request pointer to a request structure
 684  * @returns 0 on success, non-zero error on failure
 685  */
 686 static int
 687 hv_storvsc_io_request(struct storvsc_softc *sc,
 688                                           struct hv_storvsc_request *request)
 689 {
 690         struct vstor_packet *vstor_packet = &request->vstor_packet;
 691         struct vmbus_channel* outgoing_channel = NULL;
 692         int ret = 0, ch_sel;
 693
 694         vstor_packet->flags |= REQUEST_COMPLETION_FLAG;
 695
 696         vstor_packet->u.vm_srb.length =
 697             sizeof(struct vmscsi_req) - vmscsi_size_delta;
 698
 699         vstor_packet->u.vm_srb.sense_info_len = sense_buffer_size;
 700
 701         vstor_packet->u.vm_srb.transfer_len =
 702             request->prp_list.gpa_range.gpa_len;
 703
 704         vstor_packet->operation = VSTOR_OPERATION_EXECUTESRB;
 705
 706         ch_sel = (vstor_packet->u.vm_srb.lun + curcpu) % sc->hs_nchan;
 707         outgoing_channel = sc->hs_sel_chan[ch_sel];
 708
 709         mtx_unlock(&request->softc->hs_lock);
 710         if (request->prp_list.gpa_range.gpa_len) {
 711                 ret = vmbus_chan_send_prplist(outgoing_channel,
 712                     &request->prp_list.gpa_range, request->prp_cnt,
 713                     vstor_packet, VSTOR_PKT_SIZE, (uint64_t)(uintptr_t)request);
 714         } else {
 715                 ret = vmbus_chan_send(outgoing_channel,
 716                     VMBUS_CHANPKT_TYPE_INBAND, VMBUS_CHANPKT_FLAG_RC,
 717                     vstor_packet, VSTOR_PKT_SIZE, (uint64_t)(uintptr_t)request);
 718         }
 719         /* statistic for successful request sending on each channel */
 720         if (!ret) {
 721                 sc->sysctl_data.chan_send_cnt[ch_sel]++;
 722         }
 723         mtx_lock(&request->softc->hs_lock);
 724
 725         if (ret != 0) {
 726                 printf("Unable to send packet %p ret %d", vstor_packet, ret);
 727         } else {
 728                 atomic_add_int(&sc->hs_num_out_reqs, 1);
 729         }
 730
 731         return (ret);
 732 }
 733
 734
 735 /**
 736  * Process IO_COMPLETION_OPERATION and ready
 737  * the result to be completed for upper layer
 738  * processing by the CAM layer.
 739  */
 740 static void
 741 hv_storvsc_on_iocompletion(struct storvsc_softc *sc,
 742                            struct vstor_packet *vstor_packet,
 743                            struct hv_storvsc_request *request)
 744 {
 745         struct vmscsi_req *vm_srb;
 746
 747         vm_srb = &vstor_packet->u.vm_srb;
 748
 749         /*
 750          * Copy some fields of the host's response into the request structure,
 751          * because the fields will be used later in storvsc_io_done().
 752          */
 753         request->vstor_packet.u.vm_srb.scsi_status = vm_srb->scsi_status;
 754         request->vstor_packet.u.vm_srb.srb_status = vm_srb->srb_status;
 755         request->vstor_packet.u.vm_srb.transfer_len = vm_srb->transfer_len;
 756
 757         if (((vm_srb->scsi_status & 0xFF) == SCSI_STATUS_CHECK_COND) &&
 758                         (vm_srb->srb_status & SRB_STATUS_AUTOSENSE_VALID)) {
 759                 /* Autosense data available */
 760
 761                 KASSERT(vm_srb->sense_info_len <= request->sense_info_len,
 762                                 ("vm_srb->sense_info_len <= "
 763                                  "request->sense_info_len"));
 764
 765                 memcpy(request->sense_data, vm_srb->u.sense_data,
 766                         vm_srb->sense_info_len);
 767
 768                 request->sense_info_len = vm_srb->sense_info_len;
 769         }
 770
 771         /* Complete request by passing to the CAM layer */
 772         storvsc_io_done(request);
 773         atomic_subtract_int(&sc->hs_num_out_reqs, 1);
 774         if (sc->hs_drain_notify && (sc->hs_num_out_reqs == 0)) {
 775                 sema_post(&sc->hs_drain_sema);
 776         }
 777 }
 778
 779 static void
 780 hv_storvsc_rescan_target(struct storvsc_softc *sc)
 781 {
 782         path_id_t pathid;
 783         target_id_t targetid;
 784         union ccb *ccb;
 785
 786         pathid = cam_sim_path(sc->hs_sim);
 787         targetid = CAM_TARGET_WILDCARD;
 788
 789         /*
 790          * Allocate a CCB and schedule a rescan.
 791          */
 792         ccb = xpt_alloc_ccb_nowait();
 793         if (ccb == NULL) {
 794                 printf("unable to alloc CCB for rescan\n");
 795                 return;
 796         }
 797
 798         if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid,
 799             CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 800                 printf("unable to create path for rescan, pathid: %u,"
 801                     "targetid: %u\n", pathid, targetid);
 802                 xpt_free_ccb(ccb);
 803                 return;
 804         }
 805
 806         if (targetid == CAM_TARGET_WILDCARD)
 807                 ccb->ccb_h.func_code = XPT_SCAN_BUS;
 808         else
 809                 ccb->ccb_h.func_code = XPT_SCAN_TGT;
 810
 811         xpt_rescan(ccb);
 812 }
 813
 814 static void
 815 hv_storvsc_on_channel_callback(struct vmbus_channel *channel, void *xsc)
 816 {
 817         int ret = 0;
 818         struct storvsc_softc *sc = xsc;
 819         uint32_t bytes_recvd;
 820         uint64_t request_id;
 821         uint8_t packet[roundup2(sizeof(struct vstor_packet), 8)];
 822         struct hv_storvsc_request *request;
 823         struct vstor_packet *vstor_packet;
 824
 825         bytes_recvd = roundup2(VSTOR_PKT_SIZE, 8);
 826         ret = vmbus_chan_recv(channel, packet, &bytes_recvd, &request_id);
 827         KASSERT(ret != ENOBUFS, ("storvsc recvbuf is not large enough"));
 828         /* XXX check bytes_recvd to make sure that it contains enough data */
 829
 830         while ((ret == 0) && (bytes_recvd > 0)) {
 831                 request = (struct hv_storvsc_request *)(uintptr_t)request_id;
 832
 833                 if ((request == &sc->hs_init_req) ||
 834                         (request == &sc->hs_reset_req)) {
 835                         memcpy(&request->vstor_packet, packet,
 836                                    sizeof(struct vstor_packet));
 837                         sema_post(&request->synch_sema);
 838                 } else {
 839                         vstor_packet = (struct vstor_packet *)packet;
 840                         switch(vstor_packet->operation) {
 841                         case VSTOR_OPERATION_COMPLETEIO:
 842                                 if (request == NULL)
 843                                         panic("VMBUS: storvsc received a "
 844                                             "packet with NULL request id in "
 845                                             "COMPLETEIO operation.");
 846
 847                                 hv_storvsc_on_iocompletion(sc,
 848                                                         vstor_packet, request);
 849                                 break;
 850                         case VSTOR_OPERATION_REMOVEDEVICE:
 851                                 printf("VMBUS: storvsc operation %d not "
 852                                     "implemented.\n", vstor_packet->operation);
 853                                 /* TODO: implement */
 854                                 break;
 855                         case VSTOR_OPERATION_ENUMERATE_BUS:
 856                                 hv_storvsc_rescan_target(sc);
 857                                 break;
 858                         default:
 859                                 break;
 860                         }
 861                 }
 862
 863                 bytes_recvd = roundup2(VSTOR_PKT_SIZE, 8),
 864                 ret = vmbus_chan_recv(channel, packet, &bytes_recvd,
 865                     &request_id);
 866                 KASSERT(ret != ENOBUFS,
 867                     ("storvsc recvbuf is not large enough"));
 868                 /*
 869                  * XXX check bytes_recvd to make sure that it contains
 870                  * enough data
 871                  */
 872         }
 873 }
 874
 875 /**
 876  * @brief StorVSC probe function
 877  *
 878  * Device probe function.  Returns 0 if the input device is a StorVSC
 879  * device.  Otherwise, a ENXIO is returned.  If the input device is
 880  * for BlkVSC (paravirtual IDE) device and this support is disabled in
 881  * favor of the emulated ATA/IDE device, return ENXIO.
 882  *
 883  * @param a device
 884  * @returns 0 on success, ENXIO if not a matcing StorVSC device
 885  */
 886 static int
 887 storvsc_probe(device_t dev)
 888 {
 889         int ret = ENXIO;
 890
 891         switch (storvsc_get_storage_type(dev)) {
 892         case DRIVER_BLKVSC:
 893                 if(bootverbose)
 894                         device_printf(dev,
 895                             "Enlightened ATA/IDE detected\n");
 896                 device_set_desc(dev, g_drv_props_table[DRIVER_BLKVSC].drv_desc);
 897                 ret = BUS_PROBE_DEFAULT;
 898                 break;
 899         case DRIVER_STORVSC:
 900                 if(bootverbose)
 901                         device_printf(dev, "Enlightened SCSI device detected\n");
 902                 device_set_desc(dev, g_drv_props_table[DRIVER_STORVSC].drv_desc);
 903                 ret = BUS_PROBE_DEFAULT;
 904                 break;
 905         default:
 906                 ret = ENXIO;
 907         }
 908         return (ret);
 909 }
 910
 911 static void
 912 storvsc_create_chan_sel(struct storvsc_softc *sc)
 913 {
 914         struct vmbus_channel **subch;
 915         int i, nsubch;
 916
 917         sc->hs_sel_chan[0] = sc->hs_chan;
 918         nsubch = sc->hs_nchan - 1;
 919         if (nsubch == 0)
 920                 return;
 921
 922         subch = vmbus_subchan_get(sc->hs_chan, nsubch);
 923         for (i = 0; i < nsubch; i++)
 924                 sc->hs_sel_chan[i + 1] = subch[i];
 925         vmbus_subchan_rel(subch, nsubch);
 926 }
 927
 928 static int
 929 storvsc_init_requests(device_t dev)
 930 {
 931         struct storvsc_softc *sc = device_get_softc(dev);
 932         struct hv_storvsc_request *reqp;
 933         int error, i;
 934
 935         LIST_INIT(&sc->hs_free_list);
 936
 937         error = bus_dma_tag_create(
 938                 bus_get_dma_tag(dev),           /* parent */
 939                 1,                              /* alignment */
 940                 PAGE_SIZE,                      /* boundary */
 941                 BUS_SPACE_MAXADDR,              /* lowaddr */
 942                 BUS_SPACE_MAXADDR,              /* highaddr */
 943                 NULL, NULL,                     /* filter, filterarg */
 944                 STORVSC_DATA_SIZE_MAX,          /* maxsize */
 945                 STORVSC_DATA_SEGCNT_MAX,        /* nsegments */
 946                 STORVSC_DATA_SEGSZ_MAX,         /* maxsegsize */
 947                 0,                              /* flags */
 948                 NULL,                           /* lockfunc */
 949                 NULL,                           /* lockfuncarg */
 950                 &sc->storvsc_req_dtag);
 951         if (error) {
 952                 device_printf(dev, "failed to create storvsc dma tag\n");
 953                 return (error);
 954         }
 955
 956         for (i = 0; i < sc->hs_drv_props->drv_max_ios_per_target; ++i) {
 957                 reqp = malloc(sizeof(struct hv_storvsc_request),
 958                                  M_DEVBUF, M_WAITOK|M_ZERO);
 959                 reqp->softc = sc;
 960                 error = bus_dmamap_create(sc->storvsc_req_dtag, 0,
 961                                 &reqp->data_dmap);
 962                 if (error) {
 963                         device_printf(dev, "failed to allocate storvsc "
 964                             "data dmamap\n");
 965                         goto cleanup;
 966                 }
 967                 LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
 968         }
 969         return (0);
 970
 971 cleanup:
 972         while ((reqp = LIST_FIRST(&sc->hs_free_list)) != NULL) {
 973                 LIST_REMOVE(reqp, link);
 974                 bus_dmamap_destroy(sc->storvsc_req_dtag, reqp->data_dmap);
 975                 free(reqp, M_DEVBUF);
 976         }
 977         return (error);
 978 }
 979
 980 static void
 981 storvsc_sysctl(device_t dev)
 982 {
 983         struct sysctl_oid_list *child;
 984         struct sysctl_ctx_list *ctx;
 985         struct sysctl_oid *ch_tree, *chid_tree;
 986         struct storvsc_softc *sc;
 987         char name[16];
 988         int i;
 989
 990         sc = device_get_softc(dev);
 991         ctx = device_get_sysctl_ctx(dev);
 992         child = SYSCTL_CHILDREN(device_get_sysctl_tree(dev));
 993
 994         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "data_bio_cnt", CTLFLAG_RW,
 995                 &sc->sysctl_data.data_bio_cnt, "# of bio data block");
 996         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "data_vaddr_cnt", CTLFLAG_RW,
 997                 &sc->sysctl_data.data_vaddr_cnt, "# of vaddr data block");
 998         SYSCTL_ADD_ULONG(ctx, child, OID_AUTO, "data_sg_cnt", CTLFLAG_RW,
 999                 &sc->sysctl_data.data_sg_cnt, "# of sg data block");
1000
1001         /* dev.storvsc.UNIT.channel */
1002         ch_tree = SYSCTL_ADD_NODE(ctx, child, OID_AUTO, "channel",
1003                 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
1004         if (ch_tree == NULL)
1005                 return;
1006
1007         for (i = 0; i < sc->hs_nchan; i++) {
1008                 uint32_t ch_id;
1009
1010                 ch_id = vmbus_chan_id(sc->hs_sel_chan[i]);
1011                 snprintf(name, sizeof(name), "%d", ch_id);
1012                 /* dev.storvsc.UNIT.channel.CHID */
1013                 chid_tree = SYSCTL_ADD_NODE(ctx, SYSCTL_CHILDREN(ch_tree),
1014                         OID_AUTO, name, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, "");
1015                 if (chid_tree == NULL)
1016                         return;
1017                 /* dev.storvsc.UNIT.channel.CHID.send_req */
1018                 SYSCTL_ADD_ULONG(ctx, SYSCTL_CHILDREN(chid_tree), OID_AUTO,
1019                         "send_req", CTLFLAG_RD, &sc->sysctl_data.chan_send_cnt[i],
1020                         "# of request sending from this channel");
1021         }
1022 }
1023
1024 /**
1025  * @brief StorVSC attach function
1026  *
1027  * Function responsible for allocating per-device structures,
1028  * setting up CAM interfaces and scanning for available LUNs to
1029  * be used for SCSI device peripherals.
1030  *
1031  * @param a device
1032  * @returns 0 on success or an error on failure
1033  */
1034 static int
1035 storvsc_attach(device_t dev)
1036 {
1037         enum hv_storage_type stor_type;
1038         struct storvsc_softc *sc;
1039         struct cam_devq *devq;
1040         int ret, i, j;
1041         struct hv_storvsc_request *reqp;
1042         struct root_hold_token *root_mount_token = NULL;
1043         struct hv_sgl_node *sgl_node = NULL;
1044         void *tmp_buff = NULL;
1045
1046         /*
1047          * We need to serialize storvsc attach calls.
1048          */
1049         root_mount_token = root_mount_hold("storvsc");
1050
1051         sc = device_get_softc(dev);
1052         sc->hs_nchan = 1;
1053         sc->hs_chan = vmbus_get_channel(dev);
1054
1055         stor_type = storvsc_get_storage_type(dev);
1056
1057         if (stor_type == DRIVER_UNKNOWN) {
1058                 ret = ENODEV;
1059                 goto cleanup;
1060         }
1061
1062         /* fill in driver specific properties */
1063         sc->hs_drv_props = &g_drv_props_table[stor_type];
1064         sc->hs_drv_props->drv_ringbuffer_size = hv_storvsc_ringbuffer_size;
1065         sc->hs_drv_props->drv_max_ios_per_target =
1066                 MIN(STORVSC_MAX_IO, hv_storvsc_max_io);
1067         if (bootverbose) {
1068                 printf("storvsc ringbuffer size: %d, max_io: %d\n",
1069                         sc->hs_drv_props->drv_ringbuffer_size,
1070                         sc->hs_drv_props->drv_max_ios_per_target);
1071         }
1072         /* fill in device specific properties */
1073         sc->hs_unit     = device_get_unit(dev);
1074         sc->hs_dev      = dev;
1075
1076         mtx_init(&sc->hs_lock, "hvslck", NULL, MTX_DEF);
1077
1078         ret = storvsc_init_requests(dev);
1079         if (ret != 0)
1080                 goto cleanup;
1081
1082         /* create sg-list page pool */
1083         if (FALSE == g_hv_sgl_page_pool.is_init) {
1084                 g_hv_sgl_page_pool.is_init = TRUE;
1085                 LIST_INIT(&g_hv_sgl_page_pool.in_use_sgl_list);
1086                 LIST_INIT(&g_hv_sgl_page_pool.free_sgl_list);
1087
1088                 /*
1089                  * Pre-create SG list, each SG list with
1090                  * STORVSC_DATA_SEGCNT_MAX segments, each
1091                  * segment has one page buffer
1092                  */
1093                 for (i = 0; i < sc->hs_drv_props->drv_max_ios_per_target; i++) {
1094                         sgl_node = malloc(sizeof(struct hv_sgl_node),
1095                             M_DEVBUF, M_WAITOK|M_ZERO);
1096
1097                         sgl_node->sgl_data =
1098                             sglist_alloc(STORVSC_DATA_SEGCNT_MAX,
1099                             M_WAITOK|M_ZERO);
1100
1101                         for (j = 0; j < STORVSC_DATA_SEGCNT_MAX; j++) {
1102                                 tmp_buff = malloc(PAGE_SIZE,
1103                                     M_DEVBUF, M_WAITOK|M_ZERO);
1104
1105                                 sgl_node->sgl_data->sg_segs[j].ss_paddr =
1106                                     (vm_paddr_t)tmp_buff;
1107                         }
1108
1109                         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list,
1110                             sgl_node, link);
1111                 }
1112         }
1113
1114         sc->hs_destroy = FALSE;
1115         sc->hs_drain_notify = FALSE;
1116         sema_init(&sc->hs_drain_sema, 0, "Store Drain Sema");
1117
1118         ret = hv_storvsc_connect_vsp(sc);
1119         if (ret != 0) {
1120                 goto cleanup;
1121         }
1122
1123         /* Construct cpu to channel mapping */
1124         storvsc_create_chan_sel(sc);
1125
1126         /*
1127          * Create the device queue.
1128          * Hyper-V maps each target to one SCSI HBA
1129          */
1130         devq = cam_simq_alloc(sc->hs_drv_props->drv_max_ios_per_target);
1131         if (devq == NULL) {
1132                 device_printf(dev, "Failed to alloc device queue\n");
1133                 ret = ENOMEM;
1134                 goto cleanup;
1135         }
1136
1137         sc->hs_sim = cam_sim_alloc(storvsc_action,
1138                                 storvsc_poll,
1139                                 sc->hs_drv_props->drv_name,
1140                                 sc,
1141                                 sc->hs_unit,
1142                                 &sc->hs_lock, 1,
1143                                 sc->hs_drv_props->drv_max_ios_per_target,
1144                                 devq);
1145
1146         if (sc->hs_sim == NULL) {
1147                 device_printf(dev, "Failed to alloc sim\n");
1148                 cam_simq_free(devq);
1149                 ret = ENOMEM;
1150                 goto cleanup;
1151         }
1152
1153         mtx_lock(&sc->hs_lock);
1154         /* bus_id is set to 0, need to get it from VMBUS channel query? */
1155         if (xpt_bus_register(sc->hs_sim, dev, 0) != CAM_SUCCESS) {
1156                 cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
1157                 mtx_unlock(&sc->hs_lock);
1158                 device_printf(dev, "Unable to register SCSI bus\n");
1159                 ret = ENXIO;
1160                 goto cleanup;
1161         }
1162
1163         if (xpt_create_path(&sc->hs_path, /*periph*/NULL,
1164                  cam_sim_path(sc->hs_sim),
1165                 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
1166                 xpt_bus_deregister(cam_sim_path(sc->hs_sim));
1167                 cam_sim_free(sc->hs_sim, /*free_devq*/TRUE);
1168                 mtx_unlock(&sc->hs_lock);
1169                 device_printf(dev, "Unable to create path\n");
1170                 ret = ENXIO;
1171                 goto cleanup;
1172         }
1173
1174         mtx_unlock(&sc->hs_lock);
1175
1176         storvsc_sysctl(dev);
1177
1178         root_mount_rel(root_mount_token);
1179         return (0);
1180
1181
1182 cleanup:
1183         root_mount_rel(root_mount_token);
1184         while (!LIST_EMPTY(&sc->hs_free_list)) {
1185                 reqp = LIST_FIRST(&sc->hs_free_list);
1186                 LIST_REMOVE(reqp, link);
1187                 bus_dmamap_destroy(sc->storvsc_req_dtag, reqp->data_dmap);
1188                 free(reqp, M_DEVBUF);
1189         }
1190
1191         while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1192                 sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1193                 LIST_REMOVE(sgl_node, link);
1194                 for (j = 0; j < STORVSC_DATA_SEGCNT_MAX; j++) {
1195                         if (NULL !=
1196                             (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
1197                                 free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
1198                         }
1199                 }
1200                 sglist_free(sgl_node->sgl_data);
1201                 free(sgl_node, M_DEVBUF);
1202         }
1203
1204         return (ret);
1205 }
1206
1207 /**
1208  * @brief StorVSC device detach function
1209  *
1210  * This function is responsible for safely detaching a
1211  * StorVSC device.  This includes waiting for inbound responses
1212  * to complete and freeing associated per-device structures.
1213  *
1214  * @param dev a device
1215  * returns 0 on success
1216  */
1217 static int
1218 storvsc_detach(device_t dev)
1219 {
1220         struct storvsc_softc *sc = device_get_softc(dev);
1221         struct hv_storvsc_request *reqp = NULL;
1222         struct hv_sgl_node *sgl_node = NULL;
1223         int j = 0;
1224
1225         sc->hs_destroy = TRUE;
1226
1227         /*
1228          * At this point, all outbound traffic should be disabled. We
1229          * only allow inbound traffic (responses) to proceed so that
1230          * outstanding requests can be completed.
1231          */
1232
1233         sc->hs_drain_notify = TRUE;
1234         sema_wait(&sc->hs_drain_sema);
1235         sc->hs_drain_notify = FALSE;
1236
1237         /*
1238          * Since we have already drained, we don't need to busy wait.
1239          * The call to close the channel will reset the callback
1240          * under the protection of the incoming channel lock.
1241          */
1242
1243         vmbus_chan_close(sc->hs_chan);
1244
1245         mtx_lock(&sc->hs_lock);
1246         while (!LIST_EMPTY(&sc->hs_free_list)) {
1247                 reqp = LIST_FIRST(&sc->hs_free_list);
1248                 LIST_REMOVE(reqp, link);
1249                 bus_dmamap_destroy(sc->storvsc_req_dtag, reqp->data_dmap);
1250                 free(reqp, M_DEVBUF);
1251         }
1252         mtx_unlock(&sc->hs_lock);
1253
1254         while (!LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1255                 sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1256                 LIST_REMOVE(sgl_node, link);
1257                 for (j = 0; j < STORVSC_DATA_SEGCNT_MAX; j++){
1258                         if (NULL !=
1259                             (void*)sgl_node->sgl_data->sg_segs[j].ss_paddr) {
1260                                 free((void*)sgl_node->sgl_data->sg_segs[j].ss_paddr, M_DEVBUF);
1261                         }
1262                 }
1263                 sglist_free(sgl_node->sgl_data);
1264                 free(sgl_node, M_DEVBUF);
1265         }
1266
1267         return (0);
1268 }
1269
1270 #if HVS_TIMEOUT_TEST
1271 /**
1272  * @brief unit test for timed out operations
1273  *
1274  * This function provides unit testing capability to simulate
1275  * timed out operations.  Recompilation with HV_TIMEOUT_TEST=1
1276  * is required.
1277  *
1278  * @param reqp pointer to a request structure
1279  * @param opcode SCSI operation being performed
1280  * @param wait if 1, wait for I/O to complete
1281  */
1282 static void
1283 storvsc_timeout_test(struct hv_storvsc_request *reqp,
1284                 uint8_t opcode, int wait)
1285 {
1286         int ret;
1287         union ccb *ccb = reqp->ccb;
1288         struct storvsc_softc *sc = reqp->softc;
1289
1290         if (reqp->vstor_packet.vm_srb.cdb[0] != opcode) {
1291                 return;
1292         }
1293
1294         if (wait) {
1295                 mtx_lock(&reqp->event.mtx);
1296         }
1297         ret = hv_storvsc_io_request(sc, reqp);
1298         if (ret != 0) {
1299                 if (wait) {
1300                         mtx_unlock(&reqp->event.mtx);
1301                 }
1302                 printf("%s: io_request failed with %d.\n",
1303                                 __func__, ret);
1304                 ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1305                 mtx_lock(&sc->hs_lock);
1306                 storvsc_free_request(sc, reqp);
1307                 xpt_done(ccb);
1308                 mtx_unlock(&sc->hs_lock);
1309                 return;
1310         }
1311
1312         if (wait) {
1313                 xpt_print(ccb->ccb_h.path,
1314                                 "%u: %s: waiting for IO return.\n",
1315                                 ticks, __func__);
1316                 ret = cv_timedwait(&reqp->event.cv, &reqp->event.mtx, 60*hz);
1317                 mtx_unlock(&reqp->event.mtx);
1318                 xpt_print(ccb->ccb_h.path, "%u: %s: %s.\n",
1319                                 ticks, __func__, (ret == 0)?
1320                                 "IO return detected" :
1321                                 "IO return not detected");
1322                 /*
1323                  * Now both the timer handler and io done are running
1324                  * simultaneously. We want to confirm the io done always
1325                  * finishes after the timer handler exits. So reqp used by
1326                  * timer handler is not freed or stale. Do busy loop for
1327                  * another 1/10 second to make sure io done does
1328                  * wait for the timer handler to complete.
1329                  */
1330                 DELAY(100*1000);
1331                 mtx_lock(&sc->hs_lock);
1332                 xpt_print(ccb->ccb_h.path,
1333                                 "%u: %s: finishing, queue frozen %d, "
1334                                 "ccb status 0x%x scsi_status 0x%x.\n",
1335                                 ticks, __func__, sc->hs_frozen,
1336                                 ccb->ccb_h.status,
1337                                 ccb->csio.scsi_status);
1338                 mtx_unlock(&sc->hs_lock);
1339         }
1340 }
1341 #endif /* HVS_TIMEOUT_TEST */
1342
1343 #ifdef notyet
1344 /**
1345  * @brief timeout handler for requests
1346  *
1347  * This function is called as a result of a callout expiring.
1348  *
1349  * @param arg pointer to a request
1350  */
1351 static void
1352 storvsc_timeout(void *arg)
1353 {
1354         struct hv_storvsc_request *reqp = arg;
1355         struct storvsc_softc *sc = reqp->softc;
1356         union ccb *ccb = reqp->ccb;
1357
1358         if (reqp->retries == 0) {
1359                 mtx_lock(&sc->hs_lock);
1360                 xpt_print(ccb->ccb_h.path,
1361                     "%u: IO timed out (req=0x%p), wait for another %u secs.\n",
1362                     ticks, reqp, ccb->ccb_h.timeout / 1000);
1363                 cam_error_print(ccb, CAM_ESF_ALL, CAM_EPF_ALL);
1364                 mtx_unlock(&sc->hs_lock);
1365
1366                 reqp->retries++;
1367                 callout_reset_sbt(&reqp->callout, SBT_1MS * ccb->ccb_h.timeout,
1368                     0, storvsc_timeout, reqp, 0);
1369 #if HVS_TIMEOUT_TEST
1370                 storvsc_timeout_test(reqp, SEND_DIAGNOSTIC, 0);
1371 #endif
1372                 return;
1373         }
1374
1375         mtx_lock(&sc->hs_lock);
1376         xpt_print(ccb->ccb_h.path,
1377                 "%u: IO (reqp = 0x%p) did not return for %u seconds, %s.\n",
1378                 ticks, reqp, ccb->ccb_h.timeout * (reqp->retries+1) / 1000,
1379                 (sc->hs_frozen == 0)?
1380                 "freezing the queue" : "the queue is already frozen");
1381         if (sc->hs_frozen == 0) {
1382                 sc->hs_frozen = 1;
1383                 xpt_freeze_simq(xpt_path_sim(ccb->ccb_h.path), 1);
1384         }
1385         mtx_unlock(&sc->hs_lock);
1386
1387 #if HVS_TIMEOUT_TEST
1388         storvsc_timeout_test(reqp, MODE_SELECT_10, 1);
1389 #endif
1390 }
1391 #endif
1392
1393 /**
1394  * @brief StorVSC device poll function
1395  *
1396  * This function is responsible for servicing requests when
1397  * interrupts are disabled (i.e when we are dumping core.)
1398  *
1399  * @param sim a pointer to a CAM SCSI interface module
1400  */
1401 static void
1402 storvsc_poll(struct cam_sim *sim)
1403 {
1404         struct storvsc_softc *sc = cam_sim_softc(sim);
1405
1406         mtx_assert(&sc->hs_lock, MA_OWNED);
1407         mtx_unlock(&sc->hs_lock);
1408         hv_storvsc_on_channel_callback(sc->hs_chan, sc);
1409         mtx_lock(&sc->hs_lock);
1410 }
1411
1412 /**
1413  * @brief StorVSC device action function
1414  *
1415  * This function is responsible for handling SCSI operations which
1416  * are passed from the CAM layer.  The requests are in the form of
1417  * CAM control blocks which indicate the action being performed.
1418  * Not all actions require converting the request to a VSCSI protocol
1419  * message - these actions can be responded to by this driver.
1420  * Requests which are destined for a backend storage device are converted
1421  * to a VSCSI protocol message and sent on the channel connection associated
1422  * with this device.
1423  *
1424  * @param sim pointer to a CAM SCSI interface module
1425  * @param ccb pointer to a CAM control block
1426  */
1427 static void
1428 storvsc_action(struct cam_sim *sim, union ccb *ccb)
1429 {
1430         struct storvsc_softc *sc = cam_sim_softc(sim);
1431         int res;
1432
1433         mtx_assert(&sc->hs_lock, MA_OWNED);
1434         switch (ccb->ccb_h.func_code) {
1435         case XPT_PATH_INQ: {
1436                 struct ccb_pathinq *cpi = &ccb->cpi;
1437
1438                 cpi->version_num = 1;
1439                 cpi->hba_inquiry = PI_TAG_ABLE|PI_SDTR_ABLE;
1440                 cpi->target_sprt = 0;
1441                 cpi->hba_misc = PIM_NOBUSRESET;
1442                 if (hv_storvsc_use_pim_unmapped)
1443                         cpi->hba_misc |= PIM_UNMAPPED;
1444                 cpi->maxio = STORVSC_DATA_SIZE_MAX;
1445                 cpi->hba_eng_cnt = 0;
1446                 cpi->max_target = STORVSC_MAX_TARGETS;
1447                 cpi->max_lun = sc->hs_drv_props->drv_max_luns_per_target;
1448                 cpi->initiator_id = cpi->max_target;
1449                 cpi->bus_id = cam_sim_bus(sim);
1450                 cpi->base_transfer_speed = 300000;
1451                 cpi->transport = XPORT_SAS;
1452                 cpi->transport_version = 0;
1453                 cpi->protocol = PROTO_SCSI;
1454                 cpi->protocol_version = SCSI_REV_SPC2;
1455                 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
1456                 strlcpy(cpi->hba_vid, sc->hs_drv_props->drv_name, HBA_IDLEN);
1457                 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
1458                 cpi->unit_number = cam_sim_unit(sim);
1459
1460                 ccb->ccb_h.status = CAM_REQ_CMP;
1461                 xpt_done(ccb);
1462                 return;
1463         }
1464         case XPT_GET_TRAN_SETTINGS: {
1465                 struct  ccb_trans_settings *cts = &ccb->cts;
1466
1467                 cts->transport = XPORT_SAS;
1468                 cts->transport_version = 0;
1469                 cts->protocol = PROTO_SCSI;
1470                 cts->protocol_version = SCSI_REV_SPC2;
1471
1472                 /* enable tag queuing and disconnected mode */
1473                 cts->proto_specific.valid = CTS_SCSI_VALID_TQ;
1474                 cts->proto_specific.scsi.valid = CTS_SCSI_VALID_TQ;
1475                 cts->proto_specific.scsi.flags = CTS_SCSI_FLAGS_TAG_ENB;
1476                 cts->xport_specific.valid = CTS_SPI_VALID_DISC;
1477                 cts->xport_specific.spi.flags = CTS_SPI_FLAGS_DISC_ENB;
1478
1479                 ccb->ccb_h.status = CAM_REQ_CMP;
1480                 xpt_done(ccb);
1481                 return;
1482         }
1483         case XPT_SET_TRAN_SETTINGS:     {
1484                 ccb->ccb_h.status = CAM_REQ_CMP;
1485                 xpt_done(ccb);
1486                 return;
1487         }
1488         case XPT_CALC_GEOMETRY:{
1489                 cam_calc_geometry(&ccb->ccg, 1);
1490                 xpt_done(ccb);
1491                 return;
1492         }
1493         case  XPT_RESET_BUS:
1494         case  XPT_RESET_DEV:{
1495 #if HVS_HOST_RESET
1496                 if ((res = hv_storvsc_host_reset(sc)) != 0) {
1497                         xpt_print(ccb->ccb_h.path,
1498                                 "hv_storvsc_host_reset failed with %d\n", res);
1499                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1500                         xpt_done(ccb);
1501                         return;
1502                 }
1503                 ccb->ccb_h.status = CAM_REQ_CMP;
1504                 xpt_done(ccb);
1505                 return;
1506 #else
1507                 xpt_print(ccb->ccb_h.path,
1508                                   "%s reset not supported.\n",
1509                                   (ccb->ccb_h.func_code == XPT_RESET_BUS)?
1510                                   "bus" : "dev");
1511                 ccb->ccb_h.status = CAM_REQ_INVALID;
1512                 xpt_done(ccb);
1513                 return;
1514 #endif  /* HVS_HOST_RESET */
1515         }
1516         case XPT_SCSI_IO:
1517         case XPT_IMMED_NOTIFY: {
1518                 struct hv_storvsc_request *reqp = NULL;
1519                 bus_dmamap_t dmap_saved;
1520
1521                 if (ccb->csio.cdb_len == 0) {
1522                         panic("cdl_len is 0\n");
1523                 }
1524
1525                 if (LIST_EMPTY(&sc->hs_free_list)) {
1526                         ccb->ccb_h.status = CAM_REQUEUE_REQ;
1527                         if (sc->hs_frozen == 0) {
1528                                 sc->hs_frozen = 1;
1529                                 xpt_freeze_simq(sim, /* count*/1);
1530                         }
1531                         xpt_done(ccb);
1532                         return;
1533                 }
1534
1535                 reqp = LIST_FIRST(&sc->hs_free_list);
1536                 LIST_REMOVE(reqp, link);
1537
1538                 /* Save the data_dmap before reset request */
1539                 dmap_saved = reqp->data_dmap;
1540
1541                 /* XXX this is ugly */
1542                 bzero(reqp, sizeof(struct hv_storvsc_request));
1543
1544                 /* Restore necessary bits */
1545                 reqp->data_dmap = dmap_saved;
1546                 reqp->softc = sc;
1547
1548                 ccb->ccb_h.status |= CAM_SIM_QUEUED;
1549                 if ((res = create_storvsc_request(ccb, reqp)) != 0) {
1550                         ccb->ccb_h.status = CAM_REQ_INVALID;
1551                         xpt_done(ccb);
1552                         return;
1553                 }
1554
1555 #ifdef notyet
1556                 if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
1557                         callout_init(&reqp->callout, 1);
1558                         callout_reset_sbt(&reqp->callout,
1559                             SBT_1MS * ccb->ccb_h.timeout, 0,
1560                             storvsc_timeout, reqp, 0);
1561 #if HVS_TIMEOUT_TEST
1562                         cv_init(&reqp->event.cv, "storvsc timeout cv");
1563                         mtx_init(&reqp->event.mtx, "storvsc timeout mutex",
1564                                         NULL, MTX_DEF);
1565                         switch (reqp->vstor_packet.vm_srb.cdb[0]) {
1566                                 case MODE_SELECT_10:
1567                                 case SEND_DIAGNOSTIC:
1568                                         /* To have timer send the request. */
1569                                         return;
1570                                 default:
1571                                         break;
1572                         }
1573 #endif /* HVS_TIMEOUT_TEST */
1574                 }
1575 #endif
1576
1577                 if ((res = hv_storvsc_io_request(sc, reqp)) != 0) {
1578                         xpt_print(ccb->ccb_h.path,
1579                                 "hv_storvsc_io_request failed with %d\n", res);
1580                         ccb->ccb_h.status = CAM_PROVIDE_FAIL;
1581                         storvsc_free_request(sc, reqp);
1582                         xpt_done(ccb);
1583                         return;
1584                 }
1585                 return;
1586         }
1587
1588         default:
1589                 ccb->ccb_h.status = CAM_REQ_INVALID;
1590                 xpt_done(ccb);
1591                 return;
1592         }
1593 }
1594
1595 /**
1596  * @brief destroy bounce buffer
1597  *
1598  * This function is responsible for destroy a Scatter/Gather list
1599  * that create by storvsc_create_bounce_buffer()
1600  *
1601  * @param sgl- the Scatter/Gather need be destroy
1602  * @param sg_count- page count of the SG list.
1603  *
1604  */
1605 static void
1606 storvsc_destroy_bounce_buffer(struct sglist *sgl)
1607 {
1608         struct hv_sgl_node *sgl_node = NULL;
1609         if (LIST_EMPTY(&g_hv_sgl_page_pool.in_use_sgl_list)) {
1610                 printf("storvsc error: not enough in use sgl\n");
1611                 return;
1612         }
1613         sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.in_use_sgl_list);
1614         LIST_REMOVE(sgl_node, link);
1615         sgl_node->sgl_data = sgl;
1616         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.free_sgl_list, sgl_node, link);
1617 }
1618
1619 /**
1620  * @brief create bounce buffer
1621  *
1622  * This function is responsible for create a Scatter/Gather list,
1623  * which hold several pages that can be aligned with page size.
1624  *
1625  * @param seg_count- SG-list segments count
1626  * @param write - if WRITE_TYPE, set SG list page used size to 0,
1627  * otherwise set used size to page size.
1628  *
1629  * return NULL if create failed
1630  */
1631 static struct sglist *
1632 storvsc_create_bounce_buffer(uint16_t seg_count, int write)
1633 {
1634         int i = 0;
1635         struct sglist *bounce_sgl = NULL;
1636         unsigned int buf_len = ((write == WRITE_TYPE) ? 0 : PAGE_SIZE);
1637         struct hv_sgl_node *sgl_node = NULL;
1638
1639         /* get struct sglist from free_sgl_list */
1640         if (LIST_EMPTY(&g_hv_sgl_page_pool.free_sgl_list)) {
1641                 printf("storvsc error: not enough free sgl\n");
1642                 return NULL;
1643         }
1644         sgl_node = LIST_FIRST(&g_hv_sgl_page_pool.free_sgl_list);
1645         LIST_REMOVE(sgl_node, link);
1646         bounce_sgl = sgl_node->sgl_data;
1647         LIST_INSERT_HEAD(&g_hv_sgl_page_pool.in_use_sgl_list, sgl_node, link);
1648
1649         bounce_sgl->sg_maxseg = seg_count;
1650
1651         if (write == WRITE_TYPE)
1652                 bounce_sgl->sg_nseg = 0;
1653         else
1654                 bounce_sgl->sg_nseg = seg_count;
1655
1656         for (i = 0; i < seg_count; i++)
1657                 bounce_sgl->sg_segs[i].ss_len = buf_len;
1658
1659         return bounce_sgl;
1660 }
1661
1662 /**
1663  * @brief copy data from SG list to bounce buffer
1664  *
1665  * This function is responsible for copy data from one SG list's segments
1666  * to another SG list which used as bounce buffer.
1667  *
1668  * @param bounce_sgl - the destination SG list
1669  * @param orig_sgl - the segment of the source SG list.
1670  * @param orig_sgl_count - the count of segments.
1671  * @param orig_sgl_count - indicate which segment need bounce buffer,
1672  *  set 1 means need.
1673  *
1674  */
1675 static void
1676 storvsc_copy_sgl_to_bounce_buf(struct sglist *bounce_sgl,
1677                                bus_dma_segment_t *orig_sgl,
1678                                unsigned int orig_sgl_count,
1679                                uint64_t seg_bits)
1680 {
1681         int src_sgl_idx = 0;
1682
1683         for (src_sgl_idx = 0; src_sgl_idx < orig_sgl_count; src_sgl_idx++) {
1684                 if (seg_bits & (1 << src_sgl_idx)) {
1685                         memcpy((void*)bounce_sgl->sg_segs[src_sgl_idx].ss_paddr,
1686                             (void*)orig_sgl[src_sgl_idx].ds_addr,
1687                             orig_sgl[src_sgl_idx].ds_len);
1688
1689                         bounce_sgl->sg_segs[src_sgl_idx].ss_len =
1690                             orig_sgl[src_sgl_idx].ds_len;
1691                 }
1692         }
1693 }
1694
1695 /**
1696  * @brief copy data from SG list which used as bounce to another SG list
1697  *
1698  * This function is responsible for copy data from one SG list with bounce
1699  * buffer to another SG list's segments.
1700  *
1701  * @param dest_sgl - the destination SG list's segments
1702  * @param dest_sgl_count - the count of destination SG list's segment.
1703  * @param src_sgl - the source SG list.
1704  * @param seg_bits - indicate which segment used bounce buffer of src SG-list.
1705  *
1706  */
1707 void
1708 storvsc_copy_from_bounce_buf_to_sgl(bus_dma_segment_t *dest_sgl,
1709                                     unsigned int dest_sgl_count,
1710                                     struct sglist* src_sgl,
1711                                     uint64_t seg_bits)
1712 {
1713         int sgl_idx = 0;
1714
1715         for (sgl_idx = 0; sgl_idx < dest_sgl_count; sgl_idx++) {
1716                 if (seg_bits & (1 << sgl_idx)) {
1717                         memcpy((void*)(dest_sgl[sgl_idx].ds_addr),
1718                             (void*)(src_sgl->sg_segs[sgl_idx].ss_paddr),
1719                             src_sgl->sg_segs[sgl_idx].ss_len);
1720                 }
1721         }
1722 }
1723
1724 /**
1725  * @brief check SG list with bounce buffer or not
1726  *
1727  * This function is responsible for check if need bounce buffer for SG list.
1728  *
1729  * @param sgl - the SG list's segments
1730  * @param sg_count - the count of SG list's segment.
1731  * @param bits - segmengs number that need bounce buffer
1732  *
1733  * return -1 if SG list needless bounce buffer
1734  */
1735 static int
1736 storvsc_check_bounce_buffer_sgl(bus_dma_segment_t *sgl,
1737                                 unsigned int sg_count,
1738                                 uint64_t *bits)
1739 {
1740         int i = 0;
1741         int offset = 0;
1742         uint64_t phys_addr = 0;
1743         uint64_t tmp_bits = 0;
1744         boolean_t found_hole = FALSE;
1745         boolean_t pre_aligned = TRUE;
1746
1747         if (sg_count < 2){
1748                 return -1;
1749         }
1750
1751         *bits = 0;
1752
1753         phys_addr = vtophys(sgl[0].ds_addr);
1754         offset =  phys_addr - trunc_page(phys_addr);
1755
1756         if (offset != 0) {
1757                 pre_aligned = FALSE;
1758                 tmp_bits |= 1;
1759         }
1760
1761         for (i = 1; i < sg_count; i++) {
1762                 phys_addr = vtophys(sgl[i].ds_addr);
1763                 offset =  phys_addr - trunc_page(phys_addr);
1764
1765                 if (offset == 0) {
1766                         if (FALSE == pre_aligned){
1767                                 /*
1768                                  * This segment is aligned, if the previous
1769                                  * one is not aligned, find a hole
1770                                  */
1771                                 found_hole = TRUE;
1772                         }
1773                         pre_aligned = TRUE;
1774                 } else {
1775                         tmp_bits |= 1ULL << i;
1776                         if (!pre_aligned) {
1777                                 if (phys_addr != vtophys(sgl[i-1].ds_addr +
1778                                     sgl[i-1].ds_len)) {
1779                                         /*
1780                                          * Check whether connect to previous
1781                                          * segment,if not, find the hole
1782                                          */
1783                                         found_hole = TRUE;
1784                                 }
1785                         } else {
1786                                 found_hole = TRUE;
1787                         }
1788                         pre_aligned = FALSE;
1789                 }
1790         }
1791
1792         if (!found_hole) {
1793                 return (-1);
1794         } else {
1795                 *bits = tmp_bits;
1796                 return 0;
1797         }
1798 }
1799
1800 /**
1801  * Copy bus_dma segments to multiple page buffer, which requires
1802  * the pages are compact composed except for the 1st and last pages.
1803  */
1804 static void
1805 storvsc_xferbuf_prepare(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
1806 {
1807         struct hv_storvsc_request *reqp = arg;
1808         union ccb *ccb = reqp->ccb;
1809         struct ccb_scsiio *csio = &ccb->csio;
1810         struct storvsc_gpa_range *prplist;
1811         int i;
1812
1813         prplist = &reqp->prp_list;
1814         prplist->gpa_range.gpa_len = csio->dxfer_len;
1815         prplist->gpa_range.gpa_ofs = segs[0].ds_addr & PAGE_MASK;
1816
1817         for (i = 0; i < nsegs; i++) {
1818 #ifdef INVARIANTS
1819                 if (nsegs > 1) {
1820                         if (i == 0) {
1821                                 KASSERT((segs[i].ds_addr & PAGE_MASK) +
1822                                     segs[i].ds_len == PAGE_SIZE,
1823                                     ("invalid 1st page, ofs 0x%jx, len %zu",
1824                                      (uintmax_t)segs[i].ds_addr,
1825                                      segs[i].ds_len));
1826                         } else if (i == nsegs - 1) {
1827                                 KASSERT((segs[i].ds_addr & PAGE_MASK) == 0,
1828                                     ("invalid last page, ofs 0x%jx",
1829                                      (uintmax_t)segs[i].ds_addr));
1830                         } else {
1831                                 KASSERT((segs[i].ds_addr & PAGE_MASK) == 0 &&
1832                                     segs[i].ds_len == PAGE_SIZE,
1833                                     ("not a full page, ofs 0x%jx, len %zu",
1834                                      (uintmax_t)segs[i].ds_addr,
1835                                      segs[i].ds_len));
1836                         }
1837                 }
1838 #endif
1839                 prplist->gpa_page[i] = atop(segs[i].ds_addr);
1840         }
1841         reqp->prp_cnt = nsegs;
1842 }
1843
1844 /**
1845  * @brief Fill in a request structure based on a CAM control block
1846  *
1847  * Fills in a request structure based on the contents of a CAM control
1848  * block.  The request structure holds the payload information for
1849  * VSCSI protocol request.
1850  *
1851  * @param ccb pointer to a CAM contorl block
1852  * @param reqp pointer to a request structure
1853  */
1854 static int
1855 create_storvsc_request(union ccb *ccb, struct hv_storvsc_request *reqp)
1856 {
1857         struct ccb_scsiio *csio = &ccb->csio;
1858         uint64_t phys_addr;
1859         uint32_t pfn;
1860         uint64_t not_aligned_seg_bits = 0;
1861         int error;
1862
1863         /* refer to struct vmscsi_req for meanings of these two fields */
1864         reqp->vstor_packet.u.vm_srb.port =
1865                 cam_sim_unit(xpt_path_sim(ccb->ccb_h.path));
1866         reqp->vstor_packet.u.vm_srb.path_id =
1867                 cam_sim_bus(xpt_path_sim(ccb->ccb_h.path));
1868
1869         reqp->vstor_packet.u.vm_srb.target_id = ccb->ccb_h.target_id;
1870         reqp->vstor_packet.u.vm_srb.lun = ccb->ccb_h.target_lun;
1871
1872         reqp->vstor_packet.u.vm_srb.cdb_len = csio->cdb_len;
1873         if(ccb->ccb_h.flags & CAM_CDB_POINTER) {
1874                 memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_ptr,
1875                         csio->cdb_len);
1876         } else {
1877                 memcpy(&reqp->vstor_packet.u.vm_srb.u.cdb, csio->cdb_io.cdb_bytes,
1878                         csio->cdb_len);
1879         }
1880
1881         if (hv_storvsc_use_win8ext_flags) {
1882                 reqp->vstor_packet.u.vm_srb.win8_extension.time_out_value = 60;
1883                 reqp->vstor_packet.u.vm_srb.win8_extension.srb_flags |=
1884                         SRB_FLAGS_DISABLE_SYNCH_TRANSFER;
1885         }
1886         switch (ccb->ccb_h.flags & CAM_DIR_MASK) {
1887         case CAM_DIR_OUT:
1888                 reqp->vstor_packet.u.vm_srb.data_in = WRITE_TYPE;
1889                 if (hv_storvsc_use_win8ext_flags) {
1890                         reqp->vstor_packet.u.vm_srb.win8_extension.srb_flags |=
1891                                 SRB_FLAGS_DATA_OUT;
1892                 }
1893                 break;
1894         case CAM_DIR_IN:
1895                 reqp->vstor_packet.u.vm_srb.data_in = READ_TYPE;
1896                 if (hv_storvsc_use_win8ext_flags) {
1897                         reqp->vstor_packet.u.vm_srb.win8_extension.srb_flags |=
1898                                 SRB_FLAGS_DATA_IN;
1899                 }
1900                 break;
1901         case CAM_DIR_NONE:
1902                 reqp->vstor_packet.u.vm_srb.data_in = UNKNOWN_TYPE;
1903                 if (hv_storvsc_use_win8ext_flags) {
1904                         reqp->vstor_packet.u.vm_srb.win8_extension.srb_flags |=
1905                                 SRB_FLAGS_NO_DATA_TRANSFER;
1906                 }
1907                 break;
1908         default:
1909                 printf("Error: unexpected data direction: 0x%x\n",
1910                         ccb->ccb_h.flags & CAM_DIR_MASK);
1911                 return (EINVAL);
1912         }
1913
1914         reqp->sense_data     = &csio->sense_data;
1915         reqp->sense_info_len = csio->sense_len;
1916
1917         reqp->ccb = ccb;
1918
1919         if (0 == csio->dxfer_len) {
1920                 return (0);
1921         }
1922
1923         switch (ccb->ccb_h.flags & CAM_DATA_MASK) {
1924         case CAM_DATA_BIO:
1925         case CAM_DATA_VADDR:
1926                 error = bus_dmamap_load_ccb(reqp->softc->storvsc_req_dtag,
1927                     reqp->data_dmap, ccb, storvsc_xferbuf_prepare, reqp,
1928                     BUS_DMA_NOWAIT);
1929                 if (error) {
1930                         xpt_print(ccb->ccb_h.path,
1931                             "bus_dmamap_load_ccb failed: %d\n", error);
1932                         return (error);
1933                 }
1934                 if ((ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO)
1935                         reqp->softc->sysctl_data.data_bio_cnt++;
1936                 else
1937                         reqp->softc->sysctl_data.data_vaddr_cnt++;
1938                 break;
1939
1940         case CAM_DATA_SG:
1941         {
1942                 struct storvsc_gpa_range *prplist;
1943                 int i = 0;
1944                 int offset = 0;
1945                 int ret;
1946
1947                 bus_dma_segment_t *storvsc_sglist =
1948                     (bus_dma_segment_t *)ccb->csio.data_ptr;
1949                 u_int16_t storvsc_sg_count = ccb->csio.sglist_cnt;
1950
1951                 prplist = &reqp->prp_list;
1952                 prplist->gpa_range.gpa_len = csio->dxfer_len;
1953
1954                 printf("Storvsc: get SG I/O operation, %d\n",
1955                     reqp->vstor_packet.u.vm_srb.data_in);
1956
1957                 if (storvsc_sg_count > STORVSC_DATA_SEGCNT_MAX){
1958                         printf("Storvsc: %d segments is too much, "
1959                             "only support %d segments\n",
1960                             storvsc_sg_count, STORVSC_DATA_SEGCNT_MAX);
1961                         return (EINVAL);
1962                 }
1963
1964                 /*
1965                  * We create our own bounce buffer function currently. Idealy
1966                  * we should use BUS_DMA(9) framework. But with current BUS_DMA
1967                  * code there is no callback API to check the page alignment of
1968                  * middle segments before busdma can decide if a bounce buffer
1969                  * is needed for particular segment. There is callback,
1970                  * "bus_dma_filter_t *filter", but the parrameters are not
1971                  * sufficient for storvsc driver.
1972                  * TODO:
1973                  *      Add page alignment check in BUS_DMA(9) callback. Once
1974                  *      this is complete, switch the following code to use
1975                  *      BUS_DMA(9) for storvsc bounce buffer support.
1976                  */
1977                 /* check if we need to create bounce buffer */
1978                 ret = storvsc_check_bounce_buffer_sgl(storvsc_sglist,
1979                     storvsc_sg_count, &not_aligned_seg_bits);
1980                 if (ret != -1) {
1981                         reqp->bounce_sgl =
1982                             storvsc_create_bounce_buffer(storvsc_sg_count,
1983                             reqp->vstor_packet.u.vm_srb.data_in);
1984                         if (NULL == reqp->bounce_sgl) {
1985                                 printf("Storvsc_error: "
1986                                     "create bounce buffer failed.\n");
1987                                 return (ENOMEM);
1988                         }
1989
1990                         reqp->bounce_sgl_count = storvsc_sg_count;
1991                         reqp->not_aligned_seg_bits = not_aligned_seg_bits;
1992
1993                         /*
1994                          * if it is write, we need copy the original data
1995                          *to bounce buffer
1996                          */
1997                         if (WRITE_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
1998                                 storvsc_copy_sgl_to_bounce_buf(
1999                                     reqp->bounce_sgl,
2000                                     storvsc_sglist,
2001                                     storvsc_sg_count,
2002                                     reqp->not_aligned_seg_bits);
2003                         }
2004
2005                         /* transfer virtual address to physical frame number */
2006                         if (reqp->not_aligned_seg_bits & 0x1){
2007                                 phys_addr =
2008                                     vtophys(reqp->bounce_sgl->sg_segs[0].ss_paddr);
2009                         }else{
2010                                 phys_addr =
2011                                         vtophys(storvsc_sglist[0].ds_addr);
2012                         }
2013                         prplist->gpa_range.gpa_ofs = phys_addr & PAGE_MASK;
2014
2015                         pfn = phys_addr >> PAGE_SHIFT;
2016                         prplist->gpa_page[0] = pfn;
2017
2018                         for (i = 1; i < storvsc_sg_count; i++) {
2019                                 if (reqp->not_aligned_seg_bits & (1 << i)) {
2020                                         phys_addr =
2021                                             vtophys(reqp->bounce_sgl->sg_segs[i].ss_paddr);
2022                                 } else {
2023                                         phys_addr =
2024                                             vtophys(storvsc_sglist[i].ds_addr);
2025                                 }
2026
2027                                 pfn = phys_addr >> PAGE_SHIFT;
2028                                 prplist->gpa_page[i] = pfn;
2029                         }
2030                         reqp->prp_cnt = i;
2031                 } else {
2032                         phys_addr = vtophys(storvsc_sglist[0].ds_addr);
2033
2034                         prplist->gpa_range.gpa_ofs = phys_addr & PAGE_MASK;
2035
2036                         for (i = 0; i < storvsc_sg_count; i++) {
2037                                 phys_addr = vtophys(storvsc_sglist[i].ds_addr);
2038                                 pfn = phys_addr >> PAGE_SHIFT;
2039                                 prplist->gpa_page[i] = pfn;
2040                         }
2041                         reqp->prp_cnt = i;
2042
2043                         /* check the last segment cross boundary or not */
2044                         offset = phys_addr & PAGE_MASK;
2045                         if (offset) {
2046                                 /* Add one more PRP entry */
2047                                 phys_addr =
2048                                     vtophys(storvsc_sglist[i-1].ds_addr +
2049                                     PAGE_SIZE - offset);
2050                                 pfn = phys_addr >> PAGE_SHIFT;
2051                                 prplist->gpa_page[i] = pfn;
2052                                 reqp->prp_cnt++;
2053                         }
2054
2055                         reqp->bounce_sgl_count = 0;
2056                 }
2057                 reqp->softc->sysctl_data.data_sg_cnt++;
2058                 break;
2059         }
2060         default:
2061                 printf("Unknow flags: %d\n", ccb->ccb_h.flags);
2062                 return(EINVAL);
2063         }
2064
2065         return(0);
2066 }
2067
2068 static uint32_t
2069 is_scsi_valid(const struct scsi_inquiry_data *inq_data)
2070 {
2071         u_int8_t type;
2072
2073         type = SID_TYPE(inq_data);
2074         if (type == T_NODEVICE)
2075                 return (0);
2076         if (SID_QUAL(inq_data) == SID_QUAL_BAD_LU)
2077                 return (0);
2078         return (1);
2079 }
2080
2081 /**
2082  * @brief completion function before returning to CAM
2083  *
2084  * I/O process has been completed and the result needs
2085  * to be passed to the CAM layer.
2086  * Free resources related to this request.
2087  *
2088  * @param reqp pointer to a request structure
2089  */
2090 static void
2091 storvsc_io_done(struct hv_storvsc_request *reqp)
2092 {
2093         union ccb *ccb = reqp->ccb;
2094         struct ccb_scsiio *csio = &ccb->csio;
2095         struct storvsc_softc *sc = reqp->softc;
2096         struct vmscsi_req *vm_srb = &reqp->vstor_packet.u.vm_srb;
2097         bus_dma_segment_t *ori_sglist = NULL;
2098         int ori_sg_count = 0;
2099         const struct scsi_generic *cmd;
2100
2101         /* destroy bounce buffer if it is used */
2102         if (reqp->bounce_sgl_count) {
2103                 ori_sglist = (bus_dma_segment_t *)ccb->csio.data_ptr;
2104                 ori_sg_count = ccb->csio.sglist_cnt;
2105
2106                 /*
2107                  * If it is READ operation, we should copy back the data
2108                  * to original SG list.
2109                  */
2110                 if (READ_TYPE == reqp->vstor_packet.u.vm_srb.data_in) {
2111                         storvsc_copy_from_bounce_buf_to_sgl(ori_sglist,
2112                             ori_sg_count,
2113                             reqp->bounce_sgl,
2114                             reqp->not_aligned_seg_bits);
2115                 }
2116
2117                 storvsc_destroy_bounce_buffer(reqp->bounce_sgl);
2118                 reqp->bounce_sgl_count = 0;
2119         }
2120
2121         if (reqp->retries > 0) {
2122                 mtx_lock(&sc->hs_lock);
2123 #if HVS_TIMEOUT_TEST
2124                 xpt_print(ccb->ccb_h.path,
2125                         "%u: IO returned after timeout, "
2126                         "waking up timer handler if any.\n", ticks);
2127                 mtx_lock(&reqp->event.mtx);
2128                 cv_signal(&reqp->event.cv);
2129                 mtx_unlock(&reqp->event.mtx);
2130 #endif
2131                 reqp->retries = 0;
2132                 xpt_print(ccb->ccb_h.path,
2133                         "%u: IO returned after timeout, "
2134                         "stopping timer if any.\n", ticks);
2135                 mtx_unlock(&sc->hs_lock);
2136         }
2137
2138 #ifdef notyet
2139         /*
2140          * callout_drain() will wait for the timer handler to finish
2141          * if it is running. So we don't need any lock to synchronize
2142          * between this routine and the timer handler.
2143          * Note that we need to make sure reqp is not freed when timer
2144          * handler is using or will use it.
2145          */
2146         if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
2147                 callout_drain(&reqp->callout);
2148         }
2149 #endif
2150         cmd = (const struct scsi_generic *)
2151             ((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
2152              csio->cdb_io.cdb_ptr : csio->cdb_io.cdb_bytes);
2153
2154         ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
2155         ccb->ccb_h.status &= ~CAM_STATUS_MASK;
2156         int srb_status = SRB_STATUS(vm_srb->srb_status);
2157         if (vm_srb->scsi_status == SCSI_STATUS_OK) {
2158                 if (srb_status != SRB_STATUS_SUCCESS) {
2159                         /*
2160                          * If there are errors, for example, invalid LUN,
2161                          * host will inform VM through SRB status.
2162                          */
2163                         if (bootverbose) {
2164                                 if (srb_status == SRB_STATUS_INVALID_LUN) {
2165                                         xpt_print(ccb->ccb_h.path,
2166                                             "invalid LUN %d for op: %s\n",
2167                                             vm_srb->lun,
2168                                             scsi_op_desc(cmd->opcode, NULL));
2169                                 } else {
2170                                         xpt_print(ccb->ccb_h.path,
2171                                             "Unknown SRB flag: %d for op: %s\n",
2172                                             srb_status,
2173                                             scsi_op_desc(cmd->opcode, NULL));
2174                                 }
2175                         }
2176                         ccb->ccb_h.status |= CAM_DEV_NOT_THERE;
2177                 } else {
2178                         ccb->ccb_h.status |= CAM_REQ_CMP;
2179                 }
2180
2181                 if (cmd->opcode == INQUIRY &&
2182                     srb_status == SRB_STATUS_SUCCESS) {
2183                         int resp_xfer_len, resp_buf_len, data_len;
2184                         uint8_t *resp_buf = (uint8_t *)csio->data_ptr;
2185                         struct scsi_inquiry_data *inq_data =
2186                             (struct scsi_inquiry_data *)csio->data_ptr;
2187
2188                         /* Get the buffer length reported by host */
2189                         resp_xfer_len = vm_srb->transfer_len;
2190
2191                         /* Get the available buffer length */
2192                         resp_buf_len = resp_xfer_len >= 5 ? resp_buf[4] + 5 : 0;
2193                         data_len = (resp_buf_len < resp_xfer_len) ?
2194                             resp_buf_len : resp_xfer_len;
2195                         if (bootverbose && data_len >= 5) {
2196                                 xpt_print(ccb->ccb_h.path, "storvsc inquiry "
2197                                     "(%d) [%x %x %x %x %x ... ]\n", data_len,
2198                                     resp_buf[0], resp_buf[1], resp_buf[2],
2199                                     resp_buf[3], resp_buf[4]);
2200                         }
2201                         /*
2202                          * XXX: Hyper-V (since win2012r2) responses inquiry with
2203                          * unknown version (0) for GEN-2 DVD device.
2204                          * Manually set the version number to SPC3 in order to
2205                          * ask CAM to continue probing with "PROBE_REPORT_LUNS".
2206                          * see probedone() in scsi_xpt.c
2207                          */
2208                         if (SID_TYPE(inq_data) == T_CDROM &&
2209                             inq_data->version == 0 &&
2210                             (vmstor_proto_version >= VMSTOR_PROTOCOL_VERSION_WIN8)) {
2211                                 inq_data->version = SCSI_REV_SPC3;
2212                                 if (bootverbose) {
2213                                         xpt_print(ccb->ccb_h.path,
2214                                             "set version from 0 to %d\n",
2215                                             inq_data->version);
2216                                 }
2217                         }
2218                         /*
2219                          * XXX: Manually fix the wrong response returned from WS2012
2220                          */
2221                         if (!is_scsi_valid(inq_data) &&
2222                             (vmstor_proto_version == VMSTOR_PROTOCOL_VERSION_WIN8_1 ||
2223                             vmstor_proto_version == VMSTOR_PROTOCOL_VERSION_WIN8 ||
2224                             vmstor_proto_version == VMSTOR_PROTOCOL_VERSION_WIN7)) {
2225                                 if (data_len >= 4 &&
2226                                     (resp_buf[2] == 0 || resp_buf[3] == 0)) {
2227                                         resp_buf[2] = SCSI_REV_SPC3;
2228                                         resp_buf[3] = 2; // resp fmt must be 2
2229                                         if (bootverbose)
2230                                                 xpt_print(ccb->ccb_h.path,
2231                                                     "fix version and resp fmt for 0x%x\n",
2232                                                     vmstor_proto_version);
2233                                 }
2234                         } else if (data_len >= SHORT_INQUIRY_LENGTH) {
2235                                 char vendor[16];
2236
2237                                 cam_strvis(vendor, inq_data->vendor,
2238                                     sizeof(inq_data->vendor), sizeof(vendor));
2239                                 /*
2240                                  * XXX: Upgrade SPC2 to SPC3 if host is WIN8 or
2241                                  * WIN2012 R2 in order to support UNMAP feature.
2242                                  */
2243                                 if (!strncmp(vendor, "Msft", 4) &&
2244                                     SID_ANSI_REV(inq_data) == SCSI_REV_SPC2 &&
2245                                     (vmstor_proto_version ==
2246                                      VMSTOR_PROTOCOL_VERSION_WIN8_1 ||
2247                                      vmstor_proto_version ==
2248                                      VMSTOR_PROTOCOL_VERSION_WIN8)) {
2249                                         inq_data->version = SCSI_REV_SPC3;
2250                                         if (bootverbose) {
2251                                                 xpt_print(ccb->ccb_h.path,
2252                                                     "storvsc upgrades "
2253                                                     "SPC2 to SPC3\n");
2254                                         }
2255                                 }
2256                         }
2257                 }
2258         } else {
2259                 /**
2260                  * On Some Windows hosts TEST_UNIT_READY command can return
2261                  * SRB_STATUS_ERROR and sense data, for example, asc=0x3a,1
2262                  * "(Medium not present - tray closed)". This error can be
2263                  * ignored since it will be sent to host periodically.
2264                  */
2265                 boolean_t unit_not_ready = \
2266                     vm_srb->scsi_status == SCSI_STATUS_CHECK_COND &&
2267                     cmd->opcode == TEST_UNIT_READY &&
2268                     srb_status == SRB_STATUS_ERROR;
2269                 if (!unit_not_ready && bootverbose) {
2270                         mtx_lock(&sc->hs_lock);
2271                         xpt_print(ccb->ccb_h.path,
2272                                 "storvsc scsi_status = %d, srb_status = %d\n",
2273                                 vm_srb->scsi_status, srb_status);
2274                         mtx_unlock(&sc->hs_lock);
2275                 }
2276                 ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
2277         }
2278
2279         ccb->csio.scsi_status = (vm_srb->scsi_status & 0xFF);
2280         ccb->csio.resid = ccb->csio.dxfer_len - vm_srb->transfer_len;
2281
2282         if (reqp->sense_info_len != 0) {
2283                 csio->sense_resid = csio->sense_len - reqp->sense_info_len;
2284                 ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2285         }
2286
2287         mtx_lock(&sc->hs_lock);
2288         if (reqp->softc->hs_frozen == 1) {
2289                 xpt_print(ccb->ccb_h.path,
2290                         "%u: storvsc unfreezing softc 0x%p.\n",
2291                         ticks, reqp->softc);
2292                 ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2293                 reqp->softc->hs_frozen = 0;
2294         }
2295         storvsc_free_request(sc, reqp);
2296         mtx_unlock(&sc->hs_lock);
2297
2298         xpt_done_direct(ccb);
2299 }
2300
2301 /**
2302  * @brief Free a request structure
2303  *
2304  * Free a request structure by returning it to the free list
2305  *
2306  * @param sc pointer to a softc
2307  * @param reqp pointer to a request structure
2308  */
2309 static void
2310 storvsc_free_request(struct storvsc_softc *sc, struct hv_storvsc_request *reqp)
2311 {
2312
2313         LIST_INSERT_HEAD(&sc->hs_free_list, reqp, link);
2314 }
2315
2316 /**
2317  * @brief Determine type of storage device from GUID
2318  *
2319  * Using the type GUID, determine if this is a StorVSC (paravirtual
2320  * SCSI or BlkVSC (paravirtual IDE) device.
2321  *
2322  * @param dev a device
2323  * returns an enum
2324  */
2325 static enum hv_storage_type
2326 storvsc_get_storage_type(device_t dev)
2327 {
2328         device_t parent = device_get_parent(dev);
2329
2330         if (VMBUS_PROBE_GUID(parent, dev, &gBlkVscDeviceType) == 0)
2331                 return DRIVER_BLKVSC;
2332         if (VMBUS_PROBE_GUID(parent, dev, &gStorVscDeviceType) == 0)
2333                 return DRIVER_STORVSC;
2334         return DRIVER_UNKNOWN;
2335 }
2336
2337 #define PCI_VENDOR_INTEL        0x8086
2338 #define PCI_PRODUCT_PIIX4       0x7111
2339
2340 static void
2341 storvsc_ada_probe_veto(void *arg __unused, struct cam_path *path,
2342     struct ata_params *ident_buf __unused, int *veto)
2343 {
2344
2345         /*
2346          * The ATA disks are shared with the controllers managed
2347          * by this driver, so veto the ATA disks' attachment; the
2348          * ATA disks will be attached as SCSI disks once this driver
2349          * attached.
2350          */
2351         if (path->device->protocol == PROTO_ATA) {
2352                 struct ccb_pathinq cpi;
2353
2354                 xpt_path_inq(&cpi, path);
2355                 if (cpi.ccb_h.status == CAM_REQ_CMP &&
2356                     cpi.hba_vendor == PCI_VENDOR_INTEL &&
2357                     cpi.hba_device == PCI_PRODUCT_PIIX4) {
2358                         (*veto)++;
2359                         if (bootverbose) {
2360                                 xpt_print(path,
2361                                     "Disable ATA disks on "
2362                                     "simulated ATA controller (0x%04x%04x)\n",
2363                                     cpi.hba_device, cpi.hba_vendor);
2364                         }
2365                 }
2366         }
2367 }
2368
2369 static void
2370 storvsc_sysinit(void *arg __unused)
2371 {
2372         if (vm_guest == VM_GUEST_HV) {
2373                 storvsc_handler_tag = EVENTHANDLER_REGISTER(ada_probe_veto,
2374                     storvsc_ada_probe_veto, NULL, EVENTHANDLER_PRI_ANY);
2375         }
2376 }
2377 SYSINIT(storvsc_sys_init, SI_SUB_DRIVERS, SI_ORDER_SECOND, storvsc_sysinit,
2378     NULL);
2379
2380 static void
2381 storvsc_sysuninit(void *arg __unused)
2382 {
2383         if (storvsc_handler_tag != NULL)
2384                 EVENTHANDLER_DEREGISTER(ada_probe_veto, storvsc_handler_tag);
2385 }
2386 SYSUNINIT(storvsc_sys_uninit, SI_SUB_DRIVERS, SI_ORDER_SECOND,
2387     storvsc_sysuninit, NULL);